drm/amd/amdgpu : Use the MES INV_TLBS API for tlb invalidation on gfx12_1

Signed-off-by: Shaoyun Liu <shaoyun.liu@amd.com>
Reviewed-by: Prike Liang <Prike.Liang@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>

2 files changed, 78 insertions, 0 deletions

diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c b/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c
index d32e88cace6a..ef6e550ce7c3 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c
@@ -369,6 +369,34 @@ static void gmc_v12_1_flush_gpu_tlb_pasid(struct amdgpu_device *adev,
 	uint16_t queried;
 	int vmid, i;
 
+	if (adev->enable_uni_mes && adev->mes.ring[0].sched.ready &&
+	    (adev->mes.sched_version & AMDGPU_MES_VERSION_MASK) >= 0x6f) {
+		struct mes_inv_tlbs_pasid_input input = {0};
+		input.xcc_id = inst;
+		input.pasid = pasid;
+		input.flush_type = flush_type;
+
+		/* MES will invalidate hubs for the device(including slave xcc) from master, ignore request from slave */
+		if (!amdgpu_gfx_is_master_xcc(adev, inst))
+			return;
+
+		input.hub_id = AMDGPU_GFXHUB(0);
+		adev->mes.funcs->invalidate_tlbs_pasid(&adev->mes, &input);
+
+		if (all_hub) {
+			/* invalidate mm_hub */
+			if (test_bit(AMDGPU_MMHUB1(0), adev->vmhubs_mask)) {
+				input.hub_id = AMDGPU_MMHUB0(0);
+				adev->mes.funcs->invalidate_tlbs_pasid(&adev->mes, &input);
+			}
+			if (test_bit(AMDGPU_MMHUB1(0), adev->vmhubs_mask)) {
+				input.hub_id = AMDGPU_MMHUB1(0);
+				adev->mes.funcs->invalidate_tlbs_pasid(&adev->mes, &input);
+			}
+		}
+		return;
+	}
+
 	for (vmid = 1; vmid < 16; vmid++) {
 		bool valid;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c b/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c
index 17d7b1731b9c..7b8c670d0a9e 100644
--- a/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c
@@ -108,6 +108,7 @@ static const char *mes_v12_1_opcodes[] = {
 	"SET_SE_MODE",
 	"SET_GANG_SUBMIT",
 	"SET_HW_RSRC_1",
+	"INVALIDATE_TLBS",
 };
 
 static const char *mes_v12_1_misc_opcodes[] = {
@@ -854,6 +855,54 @@ static int mes_v12_1_reset_legacy_queue(struct amdgpu_mes *mes,
 }
 #endif
 
+static int mes_v12_inv_tlb_convert_hub_id(uint8_t id)
+{
+	/*
+	 * MES doesn't support invalidate gc_hub on slave xcc individually
+	 * master xcc will invalidate all gc_hub for the partition
+	 */
+	if (AMDGPU_IS_GFXHUB(id))
+		return 0;
+	else if (AMDGPU_IS_MMHUB0(id))
+		return 1;
+	else if (AMDGPU_IS_MMHUB1(id))
+		return 2;
+	return -EINVAL;
+
+}
+
+static int mes_v12_1_inv_tlbs_pasid(struct amdgpu_mes *mes,
+				    struct mes_inv_tlbs_pasid_input *input)
+{
+	union MESAPI__INV_TLBS mes_inv_tlbs;
+	int xcc_id = input->xcc_id;
+	int inst = MES_PIPE_INST(xcc_id, AMDGPU_MES_SCHED_PIPE);
+	int ret;
+
+	if (mes->enable_coop_mode)
+		xcc_id = mes->master_xcc_ids[inst];
+
+	memset(&mes_inv_tlbs, 0, sizeof(mes_inv_tlbs));
+
+	mes_inv_tlbs.header.type = MES_API_TYPE_SCHEDULER;
+	mes_inv_tlbs.header.opcode = MES_SCH_API_INV_TLBS;
+	mes_inv_tlbs.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
+
+	mes_inv_tlbs.invalidate_tlbs.inv_sel = 0;
+	mes_inv_tlbs.invalidate_tlbs.flush_type = input->flush_type;
+	mes_inv_tlbs.invalidate_tlbs.inv_sel_id = input->pasid;
+
+	/*convert amdgpu_mes_hub_id to mes expected hub_id */
+	ret = mes_v12_inv_tlb_convert_hub_id(input->hub_id);
+	if (ret < 0)
+		return -EINVAL;
+	mes_inv_tlbs.invalidate_tlbs.hub_id = ret;
+	return mes_v12_1_submit_pkt_and_poll_completion(mes, xcc_id, AMDGPU_MES_KIQ_PIPE,
+			&mes_inv_tlbs, sizeof(mes_inv_tlbs),
+			offsetof(union MESAPI__INV_TLBS, api_status));
+
+}
+
 static const struct amdgpu_mes_funcs mes_v12_1_funcs = {
 	.add_hw_queue = mes_v12_1_add_hw_queue,
 	.remove_hw_queue = mes_v12_1_remove_hw_queue,
@@ -863,6 +912,7 @@ static const struct amdgpu_mes_funcs mes_v12_1_funcs = {
 	.resume_gang = mes_v12_1_resume_gang,
 	.misc_op = mes_v12_1_misc_op,
 	.reset_hw_queue = mes_v12_1_reset_hw_queue,
+	.invalidate_tlbs_pasid = mes_v12_1_inv_tlbs_pasid,
 };
 
 static int mes_v12_1_allocate_ucode_buffer(struct amdgpu_device *adev,