Merge tag 'amd-drm-next-5.17-2021-12-02' of https://gitlab.freedesktop.org/agd5f/linux into drm-next
amd-drm-next-5.17-2021-12-02: amdgpu: - Use generic drm fb helpers - PSR fixes - Rework DCN3.1 clkmgr - DPCD 1.3 fixes - Misc display fixes can cleanups - Clock query fixes for APUs - LTTPR fixes - DSC fixes - Misc PM fixes - RAS fixes - OLED backlight fix - SRIOV fixes - Add STB (Smart Trace Buffer) for supported dGPUs - IH rework - Enable seamless boot for DCN3.01 amdkfd: - Rework more stuff around IP discovery enumeration - Further clean up of interfaces with amdgpu - SVM fixes radeon: - Indentation fixes UAPI: - Add a new KFD header that defines some of the sysfs bitfields and enums that userspace has been using for a while The corresponding bit-fields and enums in user mode are defined in https://github.com/RadeonOpenCompute/ROCT-Thunk-Interface/blob/master/include/hsakmttypes.h Signed-off-by: Dave Airlie <airlied@redhat.com> # Conflicts: # drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c From: Alex Deucher <alexander.deucher@amd.com> Link: https://patchwork.freedesktop.org/patch/msgid/20211202191643.5970-1-alexander.deucher@amd.com
This commit is contained in:
Dave Airlie
commit
f8eb96b4df
@ -966,6 +966,7 @@ F: drivers/gpu/drm/amd/include/kgd_kfd_interface.h
|
||||
F: drivers/gpu/drm/amd/include/v9_structs.h
|
||||
F: drivers/gpu/drm/amd/include/vi_structs.h
|
||||
F: include/uapi/linux/kfd_ioctl.h
|
||||
F: include/uapi/linux/kfd_sysfs.h
|
||||
|
||||
AMD SPI DRIVER
|
||||
M: Sanjay R Mehta <sanju.mehta@amd.com>
|
||||
|
||||
@ -45,7 +45,7 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \
|
||||
amdgpu_atombios.o atombios_crtc.o amdgpu_connectors.o \
|
||||
atom.o amdgpu_fence.o amdgpu_ttm.o amdgpu_object.o amdgpu_gart.o \
|
||||
amdgpu_encoders.o amdgpu_display.o amdgpu_i2c.o \
|
||||
amdgpu_fb.o amdgpu_gem.o amdgpu_ring.o \
|
||||
amdgpu_gem.o amdgpu_ring.o \
|
||||
amdgpu_cs.o amdgpu_bios.o amdgpu_benchmark.o amdgpu_test.o \
|
||||
atombios_dp.o amdgpu_afmt.o amdgpu_trace_points.o \
|
||||
atombios_encoders.o amdgpu_sa.o atombios_i2c.o \
|
||||
|
||||
@ -72,7 +72,7 @@ void amdgpu_amdkfd_device_probe(struct amdgpu_device *adev)
|
||||
if (!kfd_initialized)
|
||||
return;
|
||||
|
||||
adev->kfd.dev = kgd2kfd_probe((struct kgd_dev *)adev, vf);
|
||||
adev->kfd.dev = kgd2kfd_probe(adev, vf);
|
||||
|
||||
if (adev->kfd.dev)
|
||||
amdgpu_amdkfd_total_mem_size += adev->gmc.real_vram_size;
|
||||
@ -233,19 +233,16 @@ int amdgpu_amdkfd_post_reset(struct amdgpu_device *adev)
|
||||
return r;
|
||||
}
|
||||
|
||||
void amdgpu_amdkfd_gpu_reset(struct kgd_dev *kgd)
|
||||
void amdgpu_amdkfd_gpu_reset(struct amdgpu_device *adev)
|
||||
{
|
||||
struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
|
||||
|
||||
if (amdgpu_device_should_recover_gpu(adev))
|
||||
amdgpu_device_gpu_recover(adev, NULL);
|
||||
}
|
||||
|
||||
int amdgpu_amdkfd_alloc_gtt_mem(struct kgd_dev *kgd, size_t size,
|
||||
int amdgpu_amdkfd_alloc_gtt_mem(struct amdgpu_device *adev, size_t size,
|
||||
void **mem_obj, uint64_t *gpu_addr,
|
||||
void **cpu_ptr, bool cp_mqd_gfx9)
|
||||
{
|
||||
struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
|
||||
struct amdgpu_bo *bo = NULL;
|
||||
struct amdgpu_bo_param bp;
|
||||
int r;
|
||||
@ -314,7 +311,7 @@ allocate_mem_reserve_bo_failed:
|
||||
return r;
|
||||
}
|
||||
|
||||
void amdgpu_amdkfd_free_gtt_mem(struct kgd_dev *kgd, void *mem_obj)
|
||||
void amdgpu_amdkfd_free_gtt_mem(struct amdgpu_device *adev, void *mem_obj)
|
||||
{
|
||||
struct amdgpu_bo *bo = (struct amdgpu_bo *) mem_obj;
|
||||
|
||||
@ -325,10 +322,9 @@ void amdgpu_amdkfd_free_gtt_mem(struct kgd_dev *kgd, void *mem_obj)
|
||||
amdgpu_bo_unref(&(bo));
|
||||
}
|
||||
|
||||
int amdgpu_amdkfd_alloc_gws(struct kgd_dev *kgd, size_t size,
|
||||
int amdgpu_amdkfd_alloc_gws(struct amdgpu_device *adev, size_t size,
|
||||
void **mem_obj)
|
||||
{
|
||||
struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
|
||||
struct amdgpu_bo *bo = NULL;
|
||||
struct amdgpu_bo_user *ubo;
|
||||
struct amdgpu_bo_param bp;
|
||||
@ -355,18 +351,16 @@ int amdgpu_amdkfd_alloc_gws(struct kgd_dev *kgd, size_t size,
|
||||
return 0;
|
||||
}
|
||||
|
||||
void amdgpu_amdkfd_free_gws(struct kgd_dev *kgd, void *mem_obj)
|
||||
void amdgpu_amdkfd_free_gws(struct amdgpu_device *adev, void *mem_obj)
|
||||
{
|
||||
struct amdgpu_bo *bo = (struct amdgpu_bo *)mem_obj;
|
||||
|
||||
amdgpu_bo_unref(&bo);
|
||||
}
|
||||
|
||||
uint32_t amdgpu_amdkfd_get_fw_version(struct kgd_dev *kgd,
|
||||
uint32_t amdgpu_amdkfd_get_fw_version(struct amdgpu_device *adev,
|
||||
enum kgd_engine_type type)
|
||||
{
|
||||
struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
|
||||
|
||||
switch (type) {
|
||||
case KGD_ENGINE_PFP:
|
||||
return adev->gfx.pfp_fw_version;
|
||||
@ -399,11 +393,9 @@ uint32_t amdgpu_amdkfd_get_fw_version(struct kgd_dev *kgd,
|
||||
return 0;
|
||||
}
|
||||
|
||||
void amdgpu_amdkfd_get_local_mem_info(struct kgd_dev *kgd,
|
||||
void amdgpu_amdkfd_get_local_mem_info(struct amdgpu_device *adev,
|
||||
struct kfd_local_mem_info *mem_info)
|
||||
{
|
||||
struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
|
||||
|
||||
memset(mem_info, 0, sizeof(*mem_info));
|
||||
|
||||
mem_info->local_mem_size_public = adev->gmc.visible_vram_size;
|
||||
@ -428,19 +420,15 @@ void amdgpu_amdkfd_get_local_mem_info(struct kgd_dev *kgd,
|
||||
mem_info->mem_clk_max = 100;
|
||||
}
|
||||
|
||||
uint64_t amdgpu_amdkfd_get_gpu_clock_counter(struct kgd_dev *kgd)
|
||||
uint64_t amdgpu_amdkfd_get_gpu_clock_counter(struct amdgpu_device *adev)
|
||||
{
|
||||
struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
|
||||
|
||||
if (adev->gfx.funcs->get_gpu_clock_counter)
|
||||
return adev->gfx.funcs->get_gpu_clock_counter(adev);
|
||||
return 0;
|
||||
}
|
||||
|
||||
uint32_t amdgpu_amdkfd_get_max_engine_clock_in_mhz(struct kgd_dev *kgd)
|
||||
uint32_t amdgpu_amdkfd_get_max_engine_clock_in_mhz(struct amdgpu_device *adev)
|
||||
{
|
||||
struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
|
||||
|
||||
/* the sclk is in quantas of 10kHz */
|
||||
if (amdgpu_sriov_vf(adev))
|
||||
return adev->clock.default_sclk / 100;
|
||||
@ -450,9 +438,8 @@ uint32_t amdgpu_amdkfd_get_max_engine_clock_in_mhz(struct kgd_dev *kgd)
|
||||
return 100;
|
||||
}
|
||||
|
||||
void amdgpu_amdkfd_get_cu_info(struct kgd_dev *kgd, struct kfd_cu_info *cu_info)
|
||||
void amdgpu_amdkfd_get_cu_info(struct amdgpu_device *adev, struct kfd_cu_info *cu_info)
|
||||
{
|
||||
struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
|
||||
struct amdgpu_cu_info acu_info = adev->gfx.cu_info;
|
||||
|
||||
memset(cu_info, 0, sizeof(*cu_info));
|
||||
@ -473,13 +460,12 @@ void amdgpu_amdkfd_get_cu_info(struct kgd_dev *kgd, struct kfd_cu_info *cu_info)
|
||||
cu_info->lds_size = acu_info.lds_size;
|
||||
}
|
||||
|
||||
int amdgpu_amdkfd_get_dmabuf_info(struct kgd_dev *kgd, int dma_buf_fd,
|
||||
struct kgd_dev **dma_buf_kgd,
|
||||
int amdgpu_amdkfd_get_dmabuf_info(struct amdgpu_device *adev, int dma_buf_fd,
|
||||
struct amdgpu_device **dmabuf_adev,
|
||||
uint64_t *bo_size, void *metadata_buffer,
|
||||
size_t buffer_size, uint32_t *metadata_size,
|
||||
uint32_t *flags)
|
||||
{
|
||||
struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
|
||||
struct dma_buf *dma_buf;
|
||||
struct drm_gem_object *obj;
|
||||
struct amdgpu_bo *bo;
|
||||
@ -507,8 +493,8 @@ int amdgpu_amdkfd_get_dmabuf_info(struct kgd_dev *kgd, int dma_buf_fd,
|
||||
goto out_put;
|
||||
|
||||
r = 0;
|
||||
if (dma_buf_kgd)
|
||||
*dma_buf_kgd = (struct kgd_dev *)adev;
|
||||
if (dmabuf_adev)
|
||||
*dmabuf_adev = adev;
|
||||
if (bo_size)
|
||||
*bo_size = amdgpu_bo_size(bo);
|
||||
if (metadata_buffer)
|
||||
@ -528,32 +514,18 @@ out_put:
|
||||
return r;
|
||||
}
|
||||
|
||||
uint64_t amdgpu_amdkfd_get_vram_usage(struct kgd_dev *kgd)
|
||||
uint64_t amdgpu_amdkfd_get_vram_usage(struct amdgpu_device *adev)
|
||||
{
|
||||
struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
|
||||
struct ttm_resource_manager *vram_man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM);
|
||||
|
||||
return amdgpu_vram_mgr_usage(vram_man);
|
||||
}
|
||||
|
||||
uint64_t amdgpu_amdkfd_get_hive_id(struct kgd_dev *kgd)
|
||||
uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct amdgpu_device *dst,
|
||||
struct amdgpu_device *src)
|
||||
{
|
||||
struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
|
||||
|
||||
return adev->gmc.xgmi.hive_id;
|
||||
}
|
||||
|
||||
uint64_t amdgpu_amdkfd_get_unique_id(struct kgd_dev *kgd)
|
||||
{
|
||||
struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
|
||||
|
||||
return adev->unique_id;
|
||||
}
|
||||
|
||||
uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst, struct kgd_dev *src)
|
||||
{
|
||||
struct amdgpu_device *peer_adev = (struct amdgpu_device *)src;
|
||||
struct amdgpu_device *adev = (struct amdgpu_device *)dst;
|
||||
struct amdgpu_device *peer_adev = src;
|
||||
struct amdgpu_device *adev = dst;
|
||||
int ret = amdgpu_xgmi_get_hops_count(adev, peer_adev);
|
||||
|
||||
if (ret < 0) {
|
||||
@ -565,16 +537,18 @@ uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst, struct kgd_dev *s
|
||||
return (uint8_t)ret;
|
||||
}
|
||||
|
||||
int amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(struct kgd_dev *dst, struct kgd_dev *src, bool is_min)
|
||||
int amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(struct amdgpu_device *dst,
|
||||
struct amdgpu_device *src,
|
||||
bool is_min)
|
||||
{
|
||||
struct amdgpu_device *adev = (struct amdgpu_device *)dst, *peer_adev;
|
||||
struct amdgpu_device *adev = dst, *peer_adev;
|
||||
int num_links;
|
||||
|
||||
if (adev->asic_type != CHIP_ALDEBARAN)
|
||||
return 0;
|
||||
|
||||
if (src)
|
||||
peer_adev = (struct amdgpu_device *)src;
|
||||
peer_adev = src;
|
||||
|
||||
/* num links returns 0 for indirect peers since indirect route is unknown. */
|
||||
num_links = is_min ? 1 : amdgpu_xgmi_get_num_links(adev, peer_adev);
|
||||
@ -589,9 +563,8 @@ int amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(struct kgd_dev *dst, struct kgd_dev
|
||||
return (num_links * 16 * 25000)/BITS_PER_BYTE;
|
||||
}
|
||||
|
||||
int amdgpu_amdkfd_get_pcie_bandwidth_mbytes(struct kgd_dev *dev, bool is_min)
|
||||
int amdgpu_amdkfd_get_pcie_bandwidth_mbytes(struct amdgpu_device *adev, bool is_min)
|
||||
{
|
||||
struct amdgpu_device *adev = (struct amdgpu_device *)dev;
|
||||
int num_lanes_shift = (is_min ? ffs(adev->pm.pcie_mlw_mask) :
|
||||
fls(adev->pm.pcie_mlw_mask)) - 1;
|
||||
int gen_speed_shift = (is_min ? ffs(adev->pm.pcie_gen_mask &
|
||||
@ -647,39 +620,11 @@ int amdgpu_amdkfd_get_pcie_bandwidth_mbytes(struct kgd_dev *dev, bool is_min)
|
||||
return (num_lanes_factor * gen_speed_mbits_factor)/BITS_PER_BYTE;
|
||||
}
|
||||
|
||||
uint64_t amdgpu_amdkfd_get_mmio_remap_phys_addr(struct kgd_dev *kgd)
|
||||
{
|
||||
struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
|
||||
|
||||
return adev->rmmio_remap.bus_addr;
|
||||
}
|
||||
|
||||
uint32_t amdgpu_amdkfd_get_num_gws(struct kgd_dev *kgd)
|
||||
{
|
||||
struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
|
||||
|
||||
return adev->gds.gws_size;
|
||||
}
|
||||
|
||||
uint32_t amdgpu_amdkfd_get_asic_rev_id(struct kgd_dev *kgd)
|
||||
{
|
||||
struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
|
||||
|
||||
return adev->rev_id;
|
||||
}
|
||||
|
||||
int amdgpu_amdkfd_get_noretry(struct kgd_dev *kgd)
|
||||
{
|
||||
struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
|
||||
|
||||
return adev->gmc.noretry;
|
||||
}
|
||||
|
||||
int amdgpu_amdkfd_submit_ib(struct kgd_dev *kgd, enum kgd_engine_type engine,
|
||||
int amdgpu_amdkfd_submit_ib(struct amdgpu_device *adev,
|
||||
enum kgd_engine_type engine,
|
||||
uint32_t vmid, uint64_t gpu_addr,
|
||||
uint32_t *ib_cmd, uint32_t ib_len)
|
||||
{
|
||||
struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
|
||||
struct amdgpu_job *job;
|
||||
struct amdgpu_ib *ib;
|
||||
struct amdgpu_ring *ring;
|
||||
@ -730,10 +675,8 @@ err:
|
||||
return ret;
|
||||
}
|
||||
|
||||
void amdgpu_amdkfd_set_compute_idle(struct kgd_dev *kgd, bool idle)
|
||||
void amdgpu_amdkfd_set_compute_idle(struct amdgpu_device *adev, bool idle)
|
||||
{
|
||||
struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
|
||||
|
||||
amdgpu_dpm_switch_power_profile(adev,
|
||||
PP_SMC_POWER_PROFILE_COMPUTE,
|
||||
!idle);
|
||||
@ -747,10 +690,9 @@ bool amdgpu_amdkfd_is_kfd_vmid(struct amdgpu_device *adev, u32 vmid)
|
||||
return false;
|
||||
}
|
||||
|
||||
int amdgpu_amdkfd_flush_gpu_tlb_vmid(struct kgd_dev *kgd, uint16_t vmid)
|
||||
int amdgpu_amdkfd_flush_gpu_tlb_vmid(struct amdgpu_device *adev,
|
||||
uint16_t vmid)
|
||||
{
|
||||
struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
|
||||
|
||||
if (adev->family == AMDGPU_FAMILY_AI) {
|
||||
int i;
|
||||
|
||||
@ -763,10 +705,9 @@ int amdgpu_amdkfd_flush_gpu_tlb_vmid(struct kgd_dev *kgd, uint16_t vmid)
|
||||
return 0;
|
||||
}
|
||||
|
||||
int amdgpu_amdkfd_flush_gpu_tlb_pasid(struct kgd_dev *kgd, uint16_t pasid,
|
||||
enum TLB_FLUSH_TYPE flush_type)
|
||||
int amdgpu_amdkfd_flush_gpu_tlb_pasid(struct amdgpu_device *adev,
|
||||
uint16_t pasid, enum TLB_FLUSH_TYPE flush_type)
|
||||
{
|
||||
struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
|
||||
bool all_hub = false;
|
||||
|
||||
if (adev->family == AMDGPU_FAMILY_AI)
|
||||
@ -775,21 +716,18 @@ int amdgpu_amdkfd_flush_gpu_tlb_pasid(struct kgd_dev *kgd, uint16_t pasid,
|
||||
return amdgpu_gmc_flush_gpu_tlb_pasid(adev, pasid, flush_type, all_hub);
|
||||
}
|
||||
|
||||
bool amdgpu_amdkfd_have_atomics_support(struct kgd_dev *kgd)
|
||||
bool amdgpu_amdkfd_have_atomics_support(struct amdgpu_device *adev)
|
||||
{
|
||||
struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
|
||||
|
||||
return adev->have_atomics_support;
|
||||
}
|
||||
|
||||
void amdgpu_amdkfd_ras_poison_consumption_handler(struct kgd_dev *kgd)
|
||||
void amdgpu_amdkfd_ras_poison_consumption_handler(struct amdgpu_device *adev)
|
||||
{
|
||||
struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
|
||||
struct ras_err_data err_data = {0, 0, 0, NULL};
|
||||
|
||||
/* CPU MCA will handle page retirement if connected_to_cpu is 1 */
|
||||
if (!adev->gmc.xgmi.connected_to_cpu)
|
||||
amdgpu_umc_process_ras_data_cb(adev, &err_data, NULL);
|
||||
else
|
||||
amdgpu_amdkfd_gpu_reset(kgd);
|
||||
amdgpu_amdkfd_gpu_reset(adev);
|
||||
}
|
||||
|
||||
@ -144,14 +144,16 @@ void amdgpu_amdkfd_interrupt(struct amdgpu_device *adev,
|
||||
void amdgpu_amdkfd_device_probe(struct amdgpu_device *adev);
|
||||
void amdgpu_amdkfd_device_init(struct amdgpu_device *adev);
|
||||
void amdgpu_amdkfd_device_fini_sw(struct amdgpu_device *adev);
|
||||
int amdgpu_amdkfd_submit_ib(struct kgd_dev *kgd, enum kgd_engine_type engine,
|
||||
int amdgpu_amdkfd_submit_ib(struct amdgpu_device *adev,
|
||||
enum kgd_engine_type engine,
|
||||
uint32_t vmid, uint64_t gpu_addr,
|
||||
uint32_t *ib_cmd, uint32_t ib_len);
|
||||
void amdgpu_amdkfd_set_compute_idle(struct kgd_dev *kgd, bool idle);
|
||||
bool amdgpu_amdkfd_have_atomics_support(struct kgd_dev *kgd);
|
||||
int amdgpu_amdkfd_flush_gpu_tlb_vmid(struct kgd_dev *kgd, uint16_t vmid);
|
||||
int amdgpu_amdkfd_flush_gpu_tlb_pasid(struct kgd_dev *kgd, uint16_t pasid,
|
||||
enum TLB_FLUSH_TYPE flush_type);
|
||||
void amdgpu_amdkfd_set_compute_idle(struct amdgpu_device *adev, bool idle);
|
||||
bool amdgpu_amdkfd_have_atomics_support(struct amdgpu_device *adev);
|
||||
int amdgpu_amdkfd_flush_gpu_tlb_vmid(struct amdgpu_device *adev,
|
||||
uint16_t vmid);
|
||||
int amdgpu_amdkfd_flush_gpu_tlb_pasid(struct amdgpu_device *adev,
|
||||
uint16_t pasid, enum TLB_FLUSH_TYPE flush_type);
|
||||
|
||||
bool amdgpu_amdkfd_is_kfd_vmid(struct amdgpu_device *adev, u32 vmid);
|
||||
|
||||
@ -159,7 +161,7 @@ int amdgpu_amdkfd_pre_reset(struct amdgpu_device *adev);
|
||||
|
||||
int amdgpu_amdkfd_post_reset(struct amdgpu_device *adev);
|
||||
|
||||
void amdgpu_amdkfd_gpu_reset(struct kgd_dev *kgd);
|
||||
void amdgpu_amdkfd_gpu_reset(struct amdgpu_device *adev);
|
||||
|
||||
int amdgpu_queue_mask_bit_to_set_resource_bit(struct amdgpu_device *adev,
|
||||
int queue_bit);
|
||||
@ -198,37 +200,36 @@ int amdgpu_amdkfd_evict_userptr(struct kgd_mem *mem, struct mm_struct *mm)
|
||||
}
|
||||
#endif
|
||||
/* Shared API */
|
||||
int amdgpu_amdkfd_alloc_gtt_mem(struct kgd_dev *kgd, size_t size,
|
||||
int amdgpu_amdkfd_alloc_gtt_mem(struct amdgpu_device *adev, size_t size,
|
||||
void **mem_obj, uint64_t *gpu_addr,
|
||||
void **cpu_ptr, bool mqd_gfx9);
|
||||
void amdgpu_amdkfd_free_gtt_mem(struct kgd_dev *kgd, void *mem_obj);
|
||||
int amdgpu_amdkfd_alloc_gws(struct kgd_dev *kgd, size_t size, void **mem_obj);
|
||||
void amdgpu_amdkfd_free_gws(struct kgd_dev *kgd, void *mem_obj);
|
||||
void amdgpu_amdkfd_free_gtt_mem(struct amdgpu_device *adev, void *mem_obj);
|
||||
int amdgpu_amdkfd_alloc_gws(struct amdgpu_device *adev, size_t size,
|
||||
void **mem_obj);
|
||||
void amdgpu_amdkfd_free_gws(struct amdgpu_device *adev, void *mem_obj);
|
||||
int amdgpu_amdkfd_add_gws_to_process(void *info, void *gws, struct kgd_mem **mem);
|
||||
int amdgpu_amdkfd_remove_gws_from_process(void *info, void *mem);
|
||||
uint32_t amdgpu_amdkfd_get_fw_version(struct kgd_dev *kgd,
|
||||
uint32_t amdgpu_amdkfd_get_fw_version(struct amdgpu_device *adev,
|
||||
enum kgd_engine_type type);
|
||||
void amdgpu_amdkfd_get_local_mem_info(struct kgd_dev *kgd,
|
||||
void amdgpu_amdkfd_get_local_mem_info(struct amdgpu_device *adev,
|
||||
struct kfd_local_mem_info *mem_info);
|
||||
uint64_t amdgpu_amdkfd_get_gpu_clock_counter(struct kgd_dev *kgd);
|
||||
uint64_t amdgpu_amdkfd_get_gpu_clock_counter(struct amdgpu_device *adev);
|
||||
|
||||
uint32_t amdgpu_amdkfd_get_max_engine_clock_in_mhz(struct kgd_dev *kgd);
|
||||
void amdgpu_amdkfd_get_cu_info(struct kgd_dev *kgd, struct kfd_cu_info *cu_info);
|
||||
int amdgpu_amdkfd_get_dmabuf_info(struct kgd_dev *kgd, int dma_buf_fd,
|
||||
struct kgd_dev **dmabuf_kgd,
|
||||
uint32_t amdgpu_amdkfd_get_max_engine_clock_in_mhz(struct amdgpu_device *adev);
|
||||
void amdgpu_amdkfd_get_cu_info(struct amdgpu_device *adev,
|
||||
struct kfd_cu_info *cu_info);
|
||||
int amdgpu_amdkfd_get_dmabuf_info(struct amdgpu_device *adev, int dma_buf_fd,
|
||||
struct amdgpu_device **dmabuf_adev,
|
||||
uint64_t *bo_size, void *metadata_buffer,
|
||||
size_t buffer_size, uint32_t *metadata_size,
|
||||
uint32_t *flags);
|
||||
uint64_t amdgpu_amdkfd_get_vram_usage(struct kgd_dev *kgd);
|
||||
uint64_t amdgpu_amdkfd_get_hive_id(struct kgd_dev *kgd);
|
||||
uint64_t amdgpu_amdkfd_get_unique_id(struct kgd_dev *kgd);
|
||||
uint64_t amdgpu_amdkfd_get_mmio_remap_phys_addr(struct kgd_dev *kgd);
|
||||
uint32_t amdgpu_amdkfd_get_num_gws(struct kgd_dev *kgd);
|
||||
uint32_t amdgpu_amdkfd_get_asic_rev_id(struct kgd_dev *kgd);
|
||||
int amdgpu_amdkfd_get_noretry(struct kgd_dev *kgd);
|
||||
uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst, struct kgd_dev *src);
|
||||
int amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(struct kgd_dev *dst, struct kgd_dev *src, bool is_min);
|
||||
int amdgpu_amdkfd_get_pcie_bandwidth_mbytes(struct kgd_dev *dev, bool is_min);
|
||||
uint64_t amdgpu_amdkfd_get_vram_usage(struct amdgpu_device *adev);
|
||||
uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct amdgpu_device *dst,
|
||||
struct amdgpu_device *src);
|
||||
int amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(struct amdgpu_device *dst,
|
||||
struct amdgpu_device *src,
|
||||
bool is_min);
|
||||
int amdgpu_amdkfd_get_pcie_bandwidth_mbytes(struct amdgpu_device *adev, bool is_min);
|
||||
|
||||
/* Read user wptr from a specified user address space with page fault
|
||||
* disabled. The memory must be pinned and mapped to the hardware when
|
||||
@ -258,45 +259,54 @@ int amdgpu_amdkfd_get_pcie_bandwidth_mbytes(struct kgd_dev *dev, bool is_min);
|
||||
(&((struct amdgpu_fpriv *) \
|
||||
((struct drm_file *)(drm_priv))->driver_priv)->vm)
|
||||
|
||||
int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct kgd_dev *kgd,
|
||||
int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct amdgpu_device *adev,
|
||||
struct file *filp, u32 pasid,
|
||||
void **process_info,
|
||||
struct dma_fence **ef);
|
||||
void amdgpu_amdkfd_gpuvm_release_process_vm(struct kgd_dev *kgd, void *drm_priv);
|
||||
void amdgpu_amdkfd_gpuvm_release_process_vm(struct amdgpu_device *adev,
|
||||
void *drm_priv);
|
||||
uint64_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *drm_priv);
|
||||
int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
|
||||
struct kgd_dev *kgd, uint64_t va, uint64_t size,
|
||||
struct amdgpu_device *adev, uint64_t va, uint64_t size,
|
||||
void *drm_priv, struct kgd_mem **mem,
|
||||
uint64_t *offset, uint32_t flags);
|
||||
int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
|
||||
struct kgd_dev *kgd, struct kgd_mem *mem, void *drm_priv,
|
||||
struct amdgpu_device *adev, struct kgd_mem *mem, void *drm_priv,
|
||||
uint64_t *size);
|
||||
int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
|
||||
struct kgd_dev *kgd, struct kgd_mem *mem, void *drm_priv, bool *table_freed);
|
||||
struct amdgpu_device *adev, struct kgd_mem *mem, void *drm_priv,
|
||||
bool *table_freed);
|
||||
int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
|
||||
struct kgd_dev *kgd, struct kgd_mem *mem, void *drm_priv);
|
||||
struct amdgpu_device *adev, struct kgd_mem *mem, void *drm_priv);
|
||||
int amdgpu_amdkfd_gpuvm_sync_memory(
|
||||
struct kgd_dev *kgd, struct kgd_mem *mem, bool intr);
|
||||
int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_dev *kgd,
|
||||
struct amdgpu_device *adev, struct kgd_mem *mem, bool intr);
|
||||
int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct amdgpu_device *adev,
|
||||
struct kgd_mem *mem, void **kptr, uint64_t *size);
|
||||
void amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(struct kgd_dev *kgd, struct kgd_mem *mem);
|
||||
void amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(struct amdgpu_device *adev,
|
||||
struct kgd_mem *mem);
|
||||
|
||||
int amdgpu_amdkfd_gpuvm_restore_process_bos(void *process_info,
|
||||
struct dma_fence **ef);
|
||||
int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct kgd_dev *kgd,
|
||||
int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct amdgpu_device *adev,
|
||||
struct kfd_vm_fault_info *info);
|
||||
int amdgpu_amdkfd_gpuvm_import_dmabuf(struct kgd_dev *kgd,
|
||||
int amdgpu_amdkfd_gpuvm_import_dmabuf(struct amdgpu_device *adev,
|
||||
struct dma_buf *dmabuf,
|
||||
uint64_t va, void *drm_priv,
|
||||
struct kgd_mem **mem, uint64_t *size,
|
||||
uint64_t *mmap_offset);
|
||||
int amdgpu_amdkfd_get_tile_config(struct kgd_dev *kgd,
|
||||
int amdgpu_amdkfd_get_tile_config(struct amdgpu_device *adev,
|
||||
struct tile_config *config);
|
||||
void amdgpu_amdkfd_ras_poison_consumption_handler(struct kgd_dev *kgd);
|
||||
void amdgpu_amdkfd_ras_poison_consumption_handler(struct amdgpu_device *adev);
|
||||
#if IS_ENABLED(CONFIG_HSA_AMD)
|
||||
void amdgpu_amdkfd_gpuvm_init_mem_limits(void);
|
||||
void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev,
|
||||
struct amdgpu_vm *vm);
|
||||
|
||||
/**
|
||||
* @amdgpu_amdkfd_release_notify() - Notify KFD when GEM object is released
|
||||
*
|
||||
* Allows KFD to release its resources associated with the GEM object.
|
||||
*/
|
||||
void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo);
|
||||
void amdgpu_amdkfd_reserve_system_mem(uint64_t size);
|
||||
#else
|
||||
@ -324,7 +334,7 @@ int kgd2kfd_schedule_evict_and_restore_process(struct mm_struct *mm,
|
||||
#if IS_ENABLED(CONFIG_HSA_AMD)
|
||||
int kgd2kfd_init(void);
|
||||
void kgd2kfd_exit(void);
|
||||
struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd, bool vf);
|
||||
struct kfd_dev *kgd2kfd_probe(struct amdgpu_device *adev, bool vf);
|
||||
bool kgd2kfd_device_init(struct kfd_dev *kfd,
|
||||
struct drm_device *ddev,
|
||||
const struct kgd2kfd_shared_resources *gpu_resources);
|
||||
@ -348,7 +358,7 @@ static inline void kgd2kfd_exit(void)
|
||||
}
|
||||
|
||||
static inline
|
||||
struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd, bool vf)
|
||||
struct kfd_dev *kgd2kfd_probe(struct amdgpu_device *adev, bool vf)
|
||||
{
|
||||
return NULL;
|
||||
}
|
||||
|
||||
@ -57,11 +57,6 @@
|
||||
(*dump)[i++][1] = RREG32(addr); \
|
||||
} while (0)
|
||||
|
||||
static inline struct amdgpu_device *get_amdgpu_device(struct kgd_dev *kgd)
|
||||
{
|
||||
return (struct amdgpu_device *)kgd;
|
||||
}
|
||||
|
||||
static inline struct v9_sdma_mqd *get_sdma_mqd(void *mqd)
|
||||
{
|
||||
return (struct v9_sdma_mqd *)mqd;
|
||||
@ -123,10 +118,9 @@ static uint32_t get_sdma_rlc_reg_offset(struct amdgpu_device *adev,
|
||||
return sdma_rlc_reg_offset;
|
||||
}
|
||||
|
||||
int kgd_arcturus_hqd_sdma_load(struct kgd_dev *kgd, void *mqd,
|
||||
int kgd_arcturus_hqd_sdma_load(struct amdgpu_device *adev, void *mqd,
|
||||
uint32_t __user *wptr, struct mm_struct *mm)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
struct v9_sdma_mqd *m;
|
||||
uint32_t sdma_rlc_reg_offset;
|
||||
unsigned long end_jiffies;
|
||||
@ -193,11 +187,10 @@ int kgd_arcturus_hqd_sdma_load(struct kgd_dev *kgd, void *mqd,
|
||||
return 0;
|
||||
}
|
||||
|
||||
int kgd_arcturus_hqd_sdma_dump(struct kgd_dev *kgd,
|
||||
int kgd_arcturus_hqd_sdma_dump(struct amdgpu_device *adev,
|
||||
uint32_t engine_id, uint32_t queue_id,
|
||||
uint32_t (**dump)[2], uint32_t *n_regs)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
uint32_t sdma_rlc_reg_offset = get_sdma_rlc_reg_offset(adev,
|
||||
engine_id, queue_id);
|
||||
uint32_t i = 0, reg;
|
||||
@ -225,9 +218,9 @@ int kgd_arcturus_hqd_sdma_dump(struct kgd_dev *kgd,
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool kgd_arcturus_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd)
|
||||
bool kgd_arcturus_hqd_sdma_is_occupied(struct amdgpu_device *adev,
|
||||
void *mqd)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
struct v9_sdma_mqd *m;
|
||||
uint32_t sdma_rlc_reg_offset;
|
||||
uint32_t sdma_rlc_rb_cntl;
|
||||
@ -244,10 +237,9 @@ bool kgd_arcturus_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd)
|
||||
return false;
|
||||
}
|
||||
|
||||
int kgd_arcturus_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
|
||||
int kgd_arcturus_hqd_sdma_destroy(struct amdgpu_device *adev, void *mqd,
|
||||
unsigned int utimeout)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
struct v9_sdma_mqd *m;
|
||||
uint32_t sdma_rlc_reg_offset;
|
||||
uint32_t temp;
|
||||
|
||||
@ -20,11 +20,12 @@
|
||||
* OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
int kgd_arcturus_hqd_sdma_load(struct kgd_dev *kgd, void *mqd,
|
||||
int kgd_arcturus_hqd_sdma_load(struct amdgpu_device *adev, void *mqd,
|
||||
uint32_t __user *wptr, struct mm_struct *mm);
|
||||
int kgd_arcturus_hqd_sdma_dump(struct kgd_dev *kgd,
|
||||
int kgd_arcturus_hqd_sdma_dump(struct amdgpu_device *adev,
|
||||
uint32_t engine_id, uint32_t queue_id,
|
||||
uint32_t (**dump)[2], uint32_t *n_regs);
|
||||
bool kgd_arcturus_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd);
|
||||
int kgd_arcturus_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
|
||||
bool kgd_arcturus_hqd_sdma_is_occupied(struct amdgpu_device *adev,
|
||||
void *mqd);
|
||||
int kgd_arcturus_hqd_sdma_destroy(struct amdgpu_device *adev, void *mqd,
|
||||
unsigned int utimeout);
|
||||
|
||||
@ -39,37 +39,26 @@ enum hqd_dequeue_request_type {
|
||||
SAVE_WAVES
|
||||
};
|
||||
|
||||
static inline struct amdgpu_device *get_amdgpu_device(struct kgd_dev *kgd)
|
||||
{
|
||||
return (struct amdgpu_device *)kgd;
|
||||
}
|
||||
|
||||
static void lock_srbm(struct kgd_dev *kgd, uint32_t mec, uint32_t pipe,
|
||||
static void lock_srbm(struct amdgpu_device *adev, uint32_t mec, uint32_t pipe,
|
||||
uint32_t queue, uint32_t vmid)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
|
||||
mutex_lock(&adev->srbm_mutex);
|
||||
nv_grbm_select(adev, mec, pipe, queue, vmid);
|
||||
}
|
||||
|
||||
static void unlock_srbm(struct kgd_dev *kgd)
|
||||
static void unlock_srbm(struct amdgpu_device *adev)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
|
||||
nv_grbm_select(adev, 0, 0, 0, 0);
|
||||
mutex_unlock(&adev->srbm_mutex);
|
||||
}
|
||||
|
||||
static void acquire_queue(struct kgd_dev *kgd, uint32_t pipe_id,
|
||||
static void acquire_queue(struct amdgpu_device *adev, uint32_t pipe_id,
|
||||
uint32_t queue_id)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
|
||||
uint32_t mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
|
||||
uint32_t pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec);
|
||||
|
||||
lock_srbm(kgd, mec, pipe, queue_id, 0);
|
||||
lock_srbm(adev, mec, pipe, queue_id, 0);
|
||||
}
|
||||
|
||||
static uint64_t get_queue_mask(struct amdgpu_device *adev,
|
||||
@ -81,33 +70,29 @@ static uint64_t get_queue_mask(struct amdgpu_device *adev,
|
||||
return 1ull << bit;
|
||||
}
|
||||
|
||||
static void release_queue(struct kgd_dev *kgd)
|
||||
static void release_queue(struct amdgpu_device *adev)
|
||||
{
|
||||
unlock_srbm(kgd);
|
||||
unlock_srbm(adev);
|
||||
}
|
||||
|
||||
static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid,
|
||||
static void kgd_program_sh_mem_settings(struct amdgpu_device *adev, uint32_t vmid,
|
||||
uint32_t sh_mem_config,
|
||||
uint32_t sh_mem_ape1_base,
|
||||
uint32_t sh_mem_ape1_limit,
|
||||
uint32_t sh_mem_bases)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
|
||||
lock_srbm(kgd, 0, 0, 0, vmid);
|
||||
lock_srbm(adev, 0, 0, 0, vmid);
|
||||
|
||||
WREG32_SOC15(GC, 0, mmSH_MEM_CONFIG, sh_mem_config);
|
||||
WREG32_SOC15(GC, 0, mmSH_MEM_BASES, sh_mem_bases);
|
||||
/* APE1 no longer exists on GFX9 */
|
||||
|
||||
unlock_srbm(kgd);
|
||||
unlock_srbm(adev);
|
||||
}
|
||||
|
||||
static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, u32 pasid,
|
||||
static int kgd_set_pasid_vmid_mapping(struct amdgpu_device *adev, u32 pasid,
|
||||
unsigned int vmid)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
|
||||
/*
|
||||
* We have to assume that there is no outstanding mapping.
|
||||
* The ATC_VMID_PASID_MAPPING_UPDATE_STATUS bit could be 0 because
|
||||
@ -150,22 +135,21 @@ static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, u32 pasid,
|
||||
* but still works
|
||||
*/
|
||||
|
||||
static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id)
|
||||
static int kgd_init_interrupts(struct amdgpu_device *adev, uint32_t pipe_id)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
uint32_t mec;
|
||||
uint32_t pipe;
|
||||
|
||||
mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
|
||||
pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec);
|
||||
|
||||
lock_srbm(kgd, mec, pipe, 0, 0);
|
||||
lock_srbm(adev, mec, pipe, 0, 0);
|
||||
|
||||
WREG32_SOC15(GC, 0, mmCPC_INT_CNTL,
|
||||
CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK |
|
||||
CP_INT_CNTL_RING0__OPCODE_ERROR_INT_ENABLE_MASK);
|
||||
|
||||
unlock_srbm(kgd);
|
||||
unlock_srbm(adev);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@ -218,12 +202,11 @@ static inline struct v10_sdma_mqd *get_sdma_mqd(void *mqd)
|
||||
return (struct v10_sdma_mqd *)mqd;
|
||||
}
|
||||
|
||||
static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
|
||||
uint32_t queue_id, uint32_t __user *wptr,
|
||||
uint32_t wptr_shift, uint32_t wptr_mask,
|
||||
struct mm_struct *mm)
|
||||
static int kgd_hqd_load(struct amdgpu_device *adev, void *mqd,
|
||||
uint32_t pipe_id, uint32_t queue_id,
|
||||
uint32_t __user *wptr, uint32_t wptr_shift,
|
||||
uint32_t wptr_mask, struct mm_struct *mm)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
struct v10_compute_mqd *m;
|
||||
uint32_t *mqd_hqd;
|
||||
uint32_t reg, hqd_base, data;
|
||||
@ -231,7 +214,7 @@ static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
|
||||
m = get_mqd(mqd);
|
||||
|
||||
pr_debug("Load hqd of pipe %d queue %d\n", pipe_id, queue_id);
|
||||
acquire_queue(kgd, pipe_id, queue_id);
|
||||
acquire_queue(adev, pipe_id, queue_id);
|
||||
|
||||
/* HQD registers extend from CP_MQD_BASE_ADDR to CP_HQD_EOP_WPTR_MEM. */
|
||||
mqd_hqd = &m->cp_mqd_base_addr_lo;
|
||||
@ -296,16 +279,15 @@ static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
|
||||
data = REG_SET_FIELD(m->cp_hqd_active, CP_HQD_ACTIVE, ACTIVE, 1);
|
||||
WREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE, data);
|
||||
|
||||
release_queue(kgd);
|
||||
release_queue(adev);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int kgd_hiq_mqd_load(struct kgd_dev *kgd, void *mqd,
|
||||
static int kgd_hiq_mqd_load(struct amdgpu_device *adev, void *mqd,
|
||||
uint32_t pipe_id, uint32_t queue_id,
|
||||
uint32_t doorbell_off)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring;
|
||||
struct v10_compute_mqd *m;
|
||||
uint32_t mec, pipe;
|
||||
@ -313,7 +295,7 @@ static int kgd_hiq_mqd_load(struct kgd_dev *kgd, void *mqd,
|
||||
|
||||
m = get_mqd(mqd);
|
||||
|
||||
acquire_queue(kgd, pipe_id, queue_id);
|
||||
acquire_queue(adev, pipe_id, queue_id);
|
||||
|
||||
mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
|
||||
pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec);
|
||||
@ -349,16 +331,15 @@ static int kgd_hiq_mqd_load(struct kgd_dev *kgd, void *mqd,
|
||||
|
||||
out_unlock:
|
||||
spin_unlock(&adev->gfx.kiq.ring_lock);
|
||||
release_queue(kgd);
|
||||
release_queue(adev);
|
||||
|
||||
return r;
|
||||
}
|
||||
|
||||
static int kgd_hqd_dump(struct kgd_dev *kgd,
|
||||
static int kgd_hqd_dump(struct amdgpu_device *adev,
|
||||
uint32_t pipe_id, uint32_t queue_id,
|
||||
uint32_t (**dump)[2], uint32_t *n_regs)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
uint32_t i = 0, reg;
|
||||
#define HQD_N_REGS 56
|
||||
#define DUMP_REG(addr) do { \
|
||||
@ -372,13 +353,13 @@ static int kgd_hqd_dump(struct kgd_dev *kgd,
|
||||
if (*dump == NULL)
|
||||
return -ENOMEM;
|
||||
|
||||
acquire_queue(kgd, pipe_id, queue_id);
|
||||
acquire_queue(adev, pipe_id, queue_id);
|
||||
|
||||
for (reg = SOC15_REG_OFFSET(GC, 0, mmCP_MQD_BASE_ADDR);
|
||||
reg <= SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_HI); reg++)
|
||||
DUMP_REG(reg);
|
||||
|
||||
release_queue(kgd);
|
||||
release_queue(adev);
|
||||
|
||||
WARN_ON_ONCE(i != HQD_N_REGS);
|
||||
*n_regs = i;
|
||||
@ -386,10 +367,9 @@ static int kgd_hqd_dump(struct kgd_dev *kgd,
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd,
|
||||
static int kgd_hqd_sdma_load(struct amdgpu_device *adev, void *mqd,
|
||||
uint32_t __user *wptr, struct mm_struct *mm)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
struct v10_sdma_mqd *m;
|
||||
uint32_t sdma_rlc_reg_offset;
|
||||
unsigned long end_jiffies;
|
||||
@ -456,11 +436,10 @@ static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd,
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int kgd_hqd_sdma_dump(struct kgd_dev *kgd,
|
||||
static int kgd_hqd_sdma_dump(struct amdgpu_device *adev,
|
||||
uint32_t engine_id, uint32_t queue_id,
|
||||
uint32_t (**dump)[2], uint32_t *n_regs)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
uint32_t sdma_rlc_reg_offset = get_sdma_rlc_reg_offset(adev,
|
||||
engine_id, queue_id);
|
||||
uint32_t i = 0, reg;
|
||||
@ -488,15 +467,15 @@ static int kgd_hqd_sdma_dump(struct kgd_dev *kgd,
|
||||
return 0;
|
||||
}
|
||||
|
||||
static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address,
|
||||
uint32_t pipe_id, uint32_t queue_id)
|
||||
static bool kgd_hqd_is_occupied(struct amdgpu_device *adev,
|
||||
uint64_t queue_address, uint32_t pipe_id,
|
||||
uint32_t queue_id)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
uint32_t act;
|
||||
bool retval = false;
|
||||
uint32_t low, high;
|
||||
|
||||
acquire_queue(kgd, pipe_id, queue_id);
|
||||
acquire_queue(adev, pipe_id, queue_id);
|
||||
act = RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE);
|
||||
if (act) {
|
||||
low = lower_32_bits(queue_address >> 8);
|
||||
@ -506,13 +485,12 @@ static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address,
|
||||
high == RREG32_SOC15(GC, 0, mmCP_HQD_PQ_BASE_HI))
|
||||
retval = true;
|
||||
}
|
||||
release_queue(kgd);
|
||||
release_queue(adev);
|
||||
return retval;
|
||||
}
|
||||
|
||||
static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd)
|
||||
static bool kgd_hqd_sdma_is_occupied(struct amdgpu_device *adev, void *mqd)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
struct v10_sdma_mqd *m;
|
||||
uint32_t sdma_rlc_reg_offset;
|
||||
uint32_t sdma_rlc_rb_cntl;
|
||||
@ -529,12 +507,11 @@ static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd)
|
||||
return false;
|
||||
}
|
||||
|
||||
static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd,
|
||||
static int kgd_hqd_destroy(struct amdgpu_device *adev, void *mqd,
|
||||
enum kfd_preempt_type reset_type,
|
||||
unsigned int utimeout, uint32_t pipe_id,
|
||||
uint32_t queue_id)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
enum hqd_dequeue_request_type type;
|
||||
unsigned long end_jiffies;
|
||||
uint32_t temp;
|
||||
@ -548,7 +525,7 @@ static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd,
|
||||
int retry;
|
||||
#endif
|
||||
|
||||
acquire_queue(kgd, pipe_id, queue_id);
|
||||
acquire_queue(adev, pipe_id, queue_id);
|
||||
|
||||
if (m->cp_hqd_vmid == 0)
|
||||
WREG32_FIELD15(GC, 0, RLC_CP_SCHEDULERS, scheduler1, 0);
|
||||
@ -633,20 +610,19 @@ loop:
|
||||
break;
|
||||
if (time_after(jiffies, end_jiffies)) {
|
||||
pr_err("cp queue preemption time out.\n");
|
||||
release_queue(kgd);
|
||||
release_queue(adev);
|
||||
return -ETIME;
|
||||
}
|
||||
usleep_range(500, 1000);
|
||||
}
|
||||
|
||||
release_queue(kgd);
|
||||
release_queue(adev);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
|
||||
static int kgd_hqd_sdma_destroy(struct amdgpu_device *adev, void *mqd,
|
||||
unsigned int utimeout)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
struct v10_sdma_mqd *m;
|
||||
uint32_t sdma_rlc_reg_offset;
|
||||
uint32_t temp;
|
||||
@ -683,11 +659,10 @@ static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
|
||||
return 0;
|
||||
}
|
||||
|
||||
static bool get_atc_vmid_pasid_mapping_info(struct kgd_dev *kgd,
|
||||
static bool get_atc_vmid_pasid_mapping_info(struct amdgpu_device *adev,
|
||||
uint8_t vmid, uint16_t *p_pasid)
|
||||
{
|
||||
uint32_t value;
|
||||
struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
|
||||
|
||||
value = RREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING)
|
||||
+ vmid);
|
||||
@ -696,12 +671,12 @@ static bool get_atc_vmid_pasid_mapping_info(struct kgd_dev *kgd,
|
||||
return !!(value & ATC_VMID0_PASID_MAPPING__VALID_MASK);
|
||||
}
|
||||
|
||||
static int kgd_address_watch_disable(struct kgd_dev *kgd)
|
||||
static int kgd_address_watch_disable(struct amdgpu_device *adev)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int kgd_address_watch_execute(struct kgd_dev *kgd,
|
||||
static int kgd_address_watch_execute(struct amdgpu_device *adev,
|
||||
unsigned int watch_point_id,
|
||||
uint32_t cntl_val,
|
||||
uint32_t addr_hi,
|
||||
@ -710,11 +685,10 @@ static int kgd_address_watch_execute(struct kgd_dev *kgd,
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int kgd_wave_control_execute(struct kgd_dev *kgd,
|
||||
static int kgd_wave_control_execute(struct amdgpu_device *adev,
|
||||
uint32_t gfx_index_val,
|
||||
uint32_t sq_cmd)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
uint32_t data = 0;
|
||||
|
||||
mutex_lock(&adev->grbm_idx_mutex);
|
||||
@ -735,18 +709,16 @@ static int kgd_wave_control_execute(struct kgd_dev *kgd,
|
||||
return 0;
|
||||
}
|
||||
|
||||
static uint32_t kgd_address_watch_get_offset(struct kgd_dev *kgd,
|
||||
static uint32_t kgd_address_watch_get_offset(struct amdgpu_device *adev,
|
||||
unsigned int watch_point_id,
|
||||
unsigned int reg_offset)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void set_vm_context_page_table_base(struct kgd_dev *kgd, uint32_t vmid,
|
||||
uint64_t page_table_base)
|
||||
static void set_vm_context_page_table_base(struct amdgpu_device *adev,
|
||||
uint32_t vmid, uint64_t page_table_base)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
|
||||
if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid)) {
|
||||
pr_err("trying to set page table base for wrong VMID %u\n",
|
||||
vmid);
|
||||
@ -757,12 +729,10 @@ static void set_vm_context_page_table_base(struct kgd_dev *kgd, uint32_t vmid,
|
||||
adev->gfxhub.funcs->setup_vm_pt_regs(adev, vmid, page_table_base);
|
||||
}
|
||||
|
||||
static void program_trap_handler_settings(struct kgd_dev *kgd,
|
||||
static void program_trap_handler_settings(struct amdgpu_device *adev,
|
||||
uint32_t vmid, uint64_t tba_addr, uint64_t tma_addr)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
|
||||
lock_srbm(kgd, 0, 0, 0, vmid);
|
||||
lock_srbm(adev, 0, 0, 0, vmid);
|
||||
|
||||
/*
|
||||
* Program TBA registers
|
||||
@ -781,7 +751,7 @@ static void program_trap_handler_settings(struct kgd_dev *kgd,
|
||||
WREG32(SOC15_REG_OFFSET(GC, 0, mmSQ_SHADER_TMA_HI),
|
||||
upper_32_bits(tma_addr >> 8));
|
||||
|
||||
unlock_srbm(kgd);
|
||||
unlock_srbm(adev);
|
||||
}
|
||||
|
||||
const struct kfd2kgd_calls gfx_v10_kfd2kgd = {
|
||||
|
||||
@ -38,37 +38,26 @@ enum hqd_dequeue_request_type {
|
||||
SAVE_WAVES
|
||||
};
|
||||
|
||||
static inline struct amdgpu_device *get_amdgpu_device(struct kgd_dev *kgd)
|
||||
{
|
||||
return (struct amdgpu_device *)kgd;
|
||||
}
|
||||
|
||||
static void lock_srbm(struct kgd_dev *kgd, uint32_t mec, uint32_t pipe,
|
||||
static void lock_srbm(struct amdgpu_device *adev, uint32_t mec, uint32_t pipe,
|
||||
uint32_t queue, uint32_t vmid)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
|
||||
mutex_lock(&adev->srbm_mutex);
|
||||
nv_grbm_select(adev, mec, pipe, queue, vmid);
|
||||
}
|
||||
|
||||
static void unlock_srbm(struct kgd_dev *kgd)
|
||||
static void unlock_srbm(struct amdgpu_device *adev)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
|
||||
nv_grbm_select(adev, 0, 0, 0, 0);
|
||||
mutex_unlock(&adev->srbm_mutex);
|
||||
}
|
||||
|
||||
static void acquire_queue(struct kgd_dev *kgd, uint32_t pipe_id,
|
||||
static void acquire_queue(struct amdgpu_device *adev, uint32_t pipe_id,
|
||||
uint32_t queue_id)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
|
||||
uint32_t mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
|
||||
uint32_t pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec);
|
||||
|
||||
lock_srbm(kgd, mec, pipe, queue_id, 0);
|
||||
lock_srbm(adev, mec, pipe, queue_id, 0);
|
||||
}
|
||||
|
||||
static uint64_t get_queue_mask(struct amdgpu_device *adev,
|
||||
@ -80,34 +69,30 @@ static uint64_t get_queue_mask(struct amdgpu_device *adev,
|
||||
return 1ull << bit;
|
||||
}
|
||||
|
||||
static void release_queue(struct kgd_dev *kgd)
|
||||
static void release_queue(struct amdgpu_device *adev)
|
||||
{
|
||||
unlock_srbm(kgd);
|
||||
unlock_srbm(adev);
|
||||
}
|
||||
|
||||
static void program_sh_mem_settings_v10_3(struct kgd_dev *kgd, uint32_t vmid,
|
||||
static void program_sh_mem_settings_v10_3(struct amdgpu_device *adev, uint32_t vmid,
|
||||
uint32_t sh_mem_config,
|
||||
uint32_t sh_mem_ape1_base,
|
||||
uint32_t sh_mem_ape1_limit,
|
||||
uint32_t sh_mem_bases)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
|
||||
lock_srbm(kgd, 0, 0, 0, vmid);
|
||||
lock_srbm(adev, 0, 0, 0, vmid);
|
||||
|
||||
WREG32_SOC15(GC, 0, mmSH_MEM_CONFIG, sh_mem_config);
|
||||
WREG32_SOC15(GC, 0, mmSH_MEM_BASES, sh_mem_bases);
|
||||
/* APE1 no longer exists on GFX9 */
|
||||
|
||||
unlock_srbm(kgd);
|
||||
unlock_srbm(adev);
|
||||
}
|
||||
|
||||
/* ATC is defeatured on Sienna_Cichlid */
|
||||
static int set_pasid_vmid_mapping_v10_3(struct kgd_dev *kgd, unsigned int pasid,
|
||||
static int set_pasid_vmid_mapping_v10_3(struct amdgpu_device *adev, unsigned int pasid,
|
||||
unsigned int vmid)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
|
||||
uint32_t value = pasid << IH_VMID_0_LUT__PASID__SHIFT;
|
||||
|
||||
/* Mapping vmid to pasid also for IH block */
|
||||
@ -118,22 +103,21 @@ static int set_pasid_vmid_mapping_v10_3(struct kgd_dev *kgd, unsigned int pasid,
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int init_interrupts_v10_3(struct kgd_dev *kgd, uint32_t pipe_id)
|
||||
static int init_interrupts_v10_3(struct amdgpu_device *adev, uint32_t pipe_id)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
uint32_t mec;
|
||||
uint32_t pipe;
|
||||
|
||||
mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
|
||||
pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec);
|
||||
|
||||
lock_srbm(kgd, mec, pipe, 0, 0);
|
||||
lock_srbm(adev, mec, pipe, 0, 0);
|
||||
|
||||
WREG32_SOC15(GC, 0, mmCPC_INT_CNTL,
|
||||
CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK |
|
||||
CP_INT_CNTL_RING0__OPCODE_ERROR_INT_ENABLE_MASK);
|
||||
|
||||
unlock_srbm(kgd);
|
||||
unlock_srbm(adev);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@ -188,12 +172,11 @@ static inline struct v10_sdma_mqd *get_sdma_mqd(void *mqd)
|
||||
return (struct v10_sdma_mqd *)mqd;
|
||||
}
|
||||
|
||||
static int hqd_load_v10_3(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
|
||||
uint32_t queue_id, uint32_t __user *wptr,
|
||||
uint32_t wptr_shift, uint32_t wptr_mask,
|
||||
struct mm_struct *mm)
|
||||
static int hqd_load_v10_3(struct amdgpu_device *adev, void *mqd,
|
||||
uint32_t pipe_id, uint32_t queue_id,
|
||||
uint32_t __user *wptr, uint32_t wptr_shift,
|
||||
uint32_t wptr_mask, struct mm_struct *mm)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
struct v10_compute_mqd *m;
|
||||
uint32_t *mqd_hqd;
|
||||
uint32_t reg, hqd_base, data;
|
||||
@ -201,7 +184,7 @@ static int hqd_load_v10_3(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
|
||||
m = get_mqd(mqd);
|
||||
|
||||
pr_debug("Load hqd of pipe %d queue %d\n", pipe_id, queue_id);
|
||||
acquire_queue(kgd, pipe_id, queue_id);
|
||||
acquire_queue(adev, pipe_id, queue_id);
|
||||
|
||||
/* HIQ is set during driver init period with vmid set to 0*/
|
||||
if (m->cp_hqd_vmid == 0) {
|
||||
@ -281,16 +264,15 @@ static int hqd_load_v10_3(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
|
||||
data = REG_SET_FIELD(m->cp_hqd_active, CP_HQD_ACTIVE, ACTIVE, 1);
|
||||
WREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE, data);
|
||||
|
||||
release_queue(kgd);
|
||||
release_queue(adev);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int hiq_mqd_load_v10_3(struct kgd_dev *kgd, void *mqd,
|
||||
static int hiq_mqd_load_v10_3(struct amdgpu_device *adev, void *mqd,
|
||||
uint32_t pipe_id, uint32_t queue_id,
|
||||
uint32_t doorbell_off)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring;
|
||||
struct v10_compute_mqd *m;
|
||||
uint32_t mec, pipe;
|
||||
@ -298,7 +280,7 @@ static int hiq_mqd_load_v10_3(struct kgd_dev *kgd, void *mqd,
|
||||
|
||||
m = get_mqd(mqd);
|
||||
|
||||
acquire_queue(kgd, pipe_id, queue_id);
|
||||
acquire_queue(adev, pipe_id, queue_id);
|
||||
|
||||
mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
|
||||
pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec);
|
||||
@ -334,16 +316,15 @@ static int hiq_mqd_load_v10_3(struct kgd_dev *kgd, void *mqd,
|
||||
|
||||
out_unlock:
|
||||
spin_unlock(&adev->gfx.kiq.ring_lock);
|
||||
release_queue(kgd);
|
||||
release_queue(adev);
|
||||
|
||||
return r;
|
||||
}
|
||||
|
||||
static int hqd_dump_v10_3(struct kgd_dev *kgd,
|
||||
static int hqd_dump_v10_3(struct amdgpu_device *adev,
|
||||
uint32_t pipe_id, uint32_t queue_id,
|
||||
uint32_t (**dump)[2], uint32_t *n_regs)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
uint32_t i = 0, reg;
|
||||
#define HQD_N_REGS 56
|
||||
#define DUMP_REG(addr) do { \
|
||||
@ -357,13 +338,13 @@ static int hqd_dump_v10_3(struct kgd_dev *kgd,
|
||||
if (*dump == NULL)
|
||||
return -ENOMEM;
|
||||
|
||||
acquire_queue(kgd, pipe_id, queue_id);
|
||||
acquire_queue(adev, pipe_id, queue_id);
|
||||
|
||||
for (reg = SOC15_REG_OFFSET(GC, 0, mmCP_MQD_BASE_ADDR);
|
||||
reg <= SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_HI); reg++)
|
||||
DUMP_REG(reg);
|
||||
|
||||
release_queue(kgd);
|
||||
release_queue(adev);
|
||||
|
||||
WARN_ON_ONCE(i != HQD_N_REGS);
|
||||
*n_regs = i;
|
||||
@ -371,10 +352,9 @@ static int hqd_dump_v10_3(struct kgd_dev *kgd,
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int hqd_sdma_load_v10_3(struct kgd_dev *kgd, void *mqd,
|
||||
static int hqd_sdma_load_v10_3(struct amdgpu_device *adev, void *mqd,
|
||||
uint32_t __user *wptr, struct mm_struct *mm)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
struct v10_sdma_mqd *m;
|
||||
uint32_t sdma_rlc_reg_offset;
|
||||
unsigned long end_jiffies;
|
||||
@ -441,11 +421,10 @@ static int hqd_sdma_load_v10_3(struct kgd_dev *kgd, void *mqd,
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int hqd_sdma_dump_v10_3(struct kgd_dev *kgd,
|
||||
static int hqd_sdma_dump_v10_3(struct amdgpu_device *adev,
|
||||
uint32_t engine_id, uint32_t queue_id,
|
||||
uint32_t (**dump)[2], uint32_t *n_regs)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
uint32_t sdma_rlc_reg_offset = get_sdma_rlc_reg_offset(adev,
|
||||
engine_id, queue_id);
|
||||
uint32_t i = 0, reg;
|
||||
@ -473,15 +452,15 @@ static int hqd_sdma_dump_v10_3(struct kgd_dev *kgd,
|
||||
return 0;
|
||||
}
|
||||
|
||||
static bool hqd_is_occupied_v10_3(struct kgd_dev *kgd, uint64_t queue_address,
|
||||
uint32_t pipe_id, uint32_t queue_id)
|
||||
static bool hqd_is_occupied_v10_3(struct amdgpu_device *adev,
|
||||
uint64_t queue_address, uint32_t pipe_id,
|
||||
uint32_t queue_id)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
uint32_t act;
|
||||
bool retval = false;
|
||||
uint32_t low, high;
|
||||
|
||||
acquire_queue(kgd, pipe_id, queue_id);
|
||||
acquire_queue(adev, pipe_id, queue_id);
|
||||
act = RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE);
|
||||
if (act) {
|
||||
low = lower_32_bits(queue_address >> 8);
|
||||
@ -491,13 +470,13 @@ static bool hqd_is_occupied_v10_3(struct kgd_dev *kgd, uint64_t queue_address,
|
||||
high == RREG32_SOC15(GC, 0, mmCP_HQD_PQ_BASE_HI))
|
||||
retval = true;
|
||||
}
|
||||
release_queue(kgd);
|
||||
release_queue(adev);
|
||||
return retval;
|
||||
}
|
||||
|
||||
static bool hqd_sdma_is_occupied_v10_3(struct kgd_dev *kgd, void *mqd)
|
||||
static bool hqd_sdma_is_occupied_v10_3(struct amdgpu_device *adev,
|
||||
void *mqd)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
struct v10_sdma_mqd *m;
|
||||
uint32_t sdma_rlc_reg_offset;
|
||||
uint32_t sdma_rlc_rb_cntl;
|
||||
@ -514,18 +493,17 @@ static bool hqd_sdma_is_occupied_v10_3(struct kgd_dev *kgd, void *mqd)
|
||||
return false;
|
||||
}
|
||||
|
||||
static int hqd_destroy_v10_3(struct kgd_dev *kgd, void *mqd,
|
||||
static int hqd_destroy_v10_3(struct amdgpu_device *adev, void *mqd,
|
||||
enum kfd_preempt_type reset_type,
|
||||
unsigned int utimeout, uint32_t pipe_id,
|
||||
uint32_t queue_id)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
enum hqd_dequeue_request_type type;
|
||||
unsigned long end_jiffies;
|
||||
uint32_t temp;
|
||||
struct v10_compute_mqd *m = get_mqd(mqd);
|
||||
|
||||
acquire_queue(kgd, pipe_id, queue_id);
|
||||
acquire_queue(adev, pipe_id, queue_id);
|
||||
|
||||
if (m->cp_hqd_vmid == 0)
|
||||
WREG32_FIELD15(GC, 0, RLC_CP_SCHEDULERS, scheduler1, 0);
|
||||
@ -555,20 +533,19 @@ static int hqd_destroy_v10_3(struct kgd_dev *kgd, void *mqd,
|
||||
if (time_after(jiffies, end_jiffies)) {
|
||||
pr_err("cp queue pipe %d queue %d preemption failed\n",
|
||||
pipe_id, queue_id);
|
||||
release_queue(kgd);
|
||||
release_queue(adev);
|
||||
return -ETIME;
|
||||
}
|
||||
usleep_range(500, 1000);
|
||||
}
|
||||
|
||||
release_queue(kgd);
|
||||
release_queue(adev);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int hqd_sdma_destroy_v10_3(struct kgd_dev *kgd, void *mqd,
|
||||
static int hqd_sdma_destroy_v10_3(struct amdgpu_device *adev, void *mqd,
|
||||
unsigned int utimeout)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
struct v10_sdma_mqd *m;
|
||||
uint32_t sdma_rlc_reg_offset;
|
||||
uint32_t temp;
|
||||
@ -606,12 +583,12 @@ static int hqd_sdma_destroy_v10_3(struct kgd_dev *kgd, void *mqd,
|
||||
}
|
||||
|
||||
|
||||
static int address_watch_disable_v10_3(struct kgd_dev *kgd)
|
||||
static int address_watch_disable_v10_3(struct amdgpu_device *adev)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int address_watch_execute_v10_3(struct kgd_dev *kgd,
|
||||
static int address_watch_execute_v10_3(struct amdgpu_device *adev,
|
||||
unsigned int watch_point_id,
|
||||
uint32_t cntl_val,
|
||||
uint32_t addr_hi,
|
||||
@ -620,11 +597,10 @@ static int address_watch_execute_v10_3(struct kgd_dev *kgd,
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int wave_control_execute_v10_3(struct kgd_dev *kgd,
|
||||
static int wave_control_execute_v10_3(struct amdgpu_device *adev,
|
||||
uint32_t gfx_index_val,
|
||||
uint32_t sq_cmd)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
uint32_t data = 0;
|
||||
|
||||
mutex_lock(&adev->grbm_idx_mutex);
|
||||
@ -645,28 +621,24 @@ static int wave_control_execute_v10_3(struct kgd_dev *kgd,
|
||||
return 0;
|
||||
}
|
||||
|
||||
static uint32_t address_watch_get_offset_v10_3(struct kgd_dev *kgd,
|
||||
static uint32_t address_watch_get_offset_v10_3(struct amdgpu_device *adev,
|
||||
unsigned int watch_point_id,
|
||||
unsigned int reg_offset)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void set_vm_context_page_table_base_v10_3(struct kgd_dev *kgd, uint32_t vmid,
|
||||
uint64_t page_table_base)
|
||||
static void set_vm_context_page_table_base_v10_3(struct amdgpu_device *adev,
|
||||
uint32_t vmid, uint64_t page_table_base)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
|
||||
/* SDMA is on gfxhub as well for Navi1* series */
|
||||
adev->gfxhub.funcs->setup_vm_pt_regs(adev, vmid, page_table_base);
|
||||
}
|
||||
|
||||
static void program_trap_handler_settings_v10_3(struct kgd_dev *kgd,
|
||||
static void program_trap_handler_settings_v10_3(struct amdgpu_device *adev,
|
||||
uint32_t vmid, uint64_t tba_addr, uint64_t tma_addr)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
|
||||
lock_srbm(kgd, 0, 0, 0, vmid);
|
||||
lock_srbm(adev, 0, 0, 0, vmid);
|
||||
|
||||
/*
|
||||
* Program TBA registers
|
||||
@ -685,15 +657,14 @@ static void program_trap_handler_settings_v10_3(struct kgd_dev *kgd,
|
||||
WREG32(SOC15_REG_OFFSET(GC, 0, mmSQ_SHADER_TMA_HI),
|
||||
upper_32_bits(tma_addr >> 8));
|
||||
|
||||
unlock_srbm(kgd);
|
||||
unlock_srbm(adev);
|
||||
}
|
||||
|
||||
#if 0
|
||||
uint32_t enable_debug_trap_v10_3(struct kgd_dev *kgd,
|
||||
uint32_t enable_debug_trap_v10_3(struct amdgpu_device *adev,
|
||||
uint32_t trap_debug_wave_launch_mode,
|
||||
uint32_t vmid)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
uint32_t data = 0;
|
||||
uint32_t orig_wave_cntl_value;
|
||||
uint32_t orig_stall_vmid;
|
||||
@ -720,10 +691,8 @@ uint32_t enable_debug_trap_v10_3(struct kgd_dev *kgd,
|
||||
return 0;
|
||||
}
|
||||
|
||||
uint32_t disable_debug_trap_v10_3(struct kgd_dev *kgd)
|
||||
uint32_t disable_debug_trap_v10_3(struct amdgpu_device *adev)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
|
||||
mutex_lock(&adev->grbm_idx_mutex);
|
||||
|
||||
WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_MASK), 0);
|
||||
@ -733,11 +702,10 @@ uint32_t disable_debug_trap_v10_3(struct kgd_dev *kgd)
|
||||
return 0;
|
||||
}
|
||||
|
||||
uint32_t set_wave_launch_trap_override_v10_3(struct kgd_dev *kgd,
|
||||
uint32_t set_wave_launch_trap_override_v10_3(struct amdgpu_device *adev,
|
||||
uint32_t trap_override,
|
||||
uint32_t trap_mask)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
uint32_t data = 0;
|
||||
|
||||
mutex_lock(&adev->grbm_idx_mutex);
|
||||
@ -762,11 +730,10 @@ uint32_t set_wave_launch_trap_override_v10_3(struct kgd_dev *kgd,
|
||||
return 0;
|
||||
}
|
||||
|
||||
uint32_t set_wave_launch_mode_v10_3(struct kgd_dev *kgd,
|
||||
uint32_t set_wave_launch_mode_v10_3(struct amdgpu_device *adev,
|
||||
uint8_t wave_launch_mode,
|
||||
uint32_t vmid)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
uint32_t data = 0;
|
||||
bool is_stall_mode;
|
||||
bool is_mode_set;
|
||||
@ -805,16 +772,14 @@ uint32_t set_wave_launch_mode_v10_3(struct kgd_dev *kgd,
|
||||
* sem_rearm_wait_time -- Wait Count for Semaphore re-arm.
|
||||
* deq_retry_wait_time -- Wait Count for Global Wave Syncs.
|
||||
*/
|
||||
void get_iq_wait_times_v10_3(struct kgd_dev *kgd,
|
||||
void get_iq_wait_times_v10_3(struct amdgpu_device *adev,
|
||||
uint32_t *wait_times)
|
||||
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
|
||||
*wait_times = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_IQ_WAIT_TIME2));
|
||||
}
|
||||
|
||||
void build_grace_period_packet_info_v10_3(struct kgd_dev *kgd,
|
||||
void build_grace_period_packet_info_v10_3(struct amdgpu_device *adev,
|
||||
uint32_t wait_times,
|
||||
uint32_t grace_period,
|
||||
uint32_t *reg_offset,
|
||||
|
||||
@ -82,68 +82,54 @@ union TCP_WATCH_CNTL_BITS {
|
||||
float f32All;
|
||||
};
|
||||
|
||||
static inline struct amdgpu_device *get_amdgpu_device(struct kgd_dev *kgd)
|
||||
{
|
||||
return (struct amdgpu_device *)kgd;
|
||||
}
|
||||
|
||||
static void lock_srbm(struct kgd_dev *kgd, uint32_t mec, uint32_t pipe,
|
||||
static void lock_srbm(struct amdgpu_device *adev, uint32_t mec, uint32_t pipe,
|
||||
uint32_t queue, uint32_t vmid)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
uint32_t value = PIPEID(pipe) | MEID(mec) | VMID(vmid) | QUEUEID(queue);
|
||||
|
||||
mutex_lock(&adev->srbm_mutex);
|
||||
WREG32(mmSRBM_GFX_CNTL, value);
|
||||
}
|
||||
|
||||
static void unlock_srbm(struct kgd_dev *kgd)
|
||||
static void unlock_srbm(struct amdgpu_device *adev)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
|
||||
WREG32(mmSRBM_GFX_CNTL, 0);
|
||||
mutex_unlock(&adev->srbm_mutex);
|
||||
}
|
||||
|
||||
static void acquire_queue(struct kgd_dev *kgd, uint32_t pipe_id,
|
||||
static void acquire_queue(struct amdgpu_device *adev, uint32_t pipe_id,
|
||||
uint32_t queue_id)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
|
||||
uint32_t mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
|
||||
uint32_t pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec);
|
||||
|
||||
lock_srbm(kgd, mec, pipe, queue_id, 0);
|
||||
lock_srbm(adev, mec, pipe, queue_id, 0);
|
||||
}
|
||||
|
||||
static void release_queue(struct kgd_dev *kgd)
|
||||
static void release_queue(struct amdgpu_device *adev)
|
||||
{
|
||||
unlock_srbm(kgd);
|
||||
unlock_srbm(adev);
|
||||
}
|
||||
|
||||
static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid,
|
||||
static void kgd_program_sh_mem_settings(struct amdgpu_device *adev, uint32_t vmid,
|
||||
uint32_t sh_mem_config,
|
||||
uint32_t sh_mem_ape1_base,
|
||||
uint32_t sh_mem_ape1_limit,
|
||||
uint32_t sh_mem_bases)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
|
||||
lock_srbm(kgd, 0, 0, 0, vmid);
|
||||
lock_srbm(adev, 0, 0, 0, vmid);
|
||||
|
||||
WREG32(mmSH_MEM_CONFIG, sh_mem_config);
|
||||
WREG32(mmSH_MEM_APE1_BASE, sh_mem_ape1_base);
|
||||
WREG32(mmSH_MEM_APE1_LIMIT, sh_mem_ape1_limit);
|
||||
WREG32(mmSH_MEM_BASES, sh_mem_bases);
|
||||
|
||||
unlock_srbm(kgd);
|
||||
unlock_srbm(adev);
|
||||
}
|
||||
|
||||
static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, u32 pasid,
|
||||
static int kgd_set_pasid_vmid_mapping(struct amdgpu_device *adev, u32 pasid,
|
||||
unsigned int vmid)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
|
||||
/*
|
||||
* We have to assume that there is no outstanding mapping.
|
||||
* The ATC_VMID_PASID_MAPPING_UPDATE_STATUS bit could be 0 because
|
||||
@ -165,21 +151,20 @@ static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, u32 pasid,
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id)
|
||||
static int kgd_init_interrupts(struct amdgpu_device *adev, uint32_t pipe_id)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
uint32_t mec;
|
||||
uint32_t pipe;
|
||||
|
||||
mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
|
||||
pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec);
|
||||
|
||||
lock_srbm(kgd, mec, pipe, 0, 0);
|
||||
lock_srbm(adev, mec, pipe, 0, 0);
|
||||
|
||||
WREG32(mmCPC_INT_CNTL, CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK |
|
||||
CP_INT_CNTL_RING0__OPCODE_ERROR_INT_ENABLE_MASK);
|
||||
|
||||
unlock_srbm(kgd);
|
||||
unlock_srbm(adev);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@ -207,12 +192,11 @@ static inline struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd)
|
||||
return (struct cik_sdma_rlc_registers *)mqd;
|
||||
}
|
||||
|
||||
static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
|
||||
uint32_t queue_id, uint32_t __user *wptr,
|
||||
uint32_t wptr_shift, uint32_t wptr_mask,
|
||||
struct mm_struct *mm)
|
||||
static int kgd_hqd_load(struct amdgpu_device *adev, void *mqd,
|
||||
uint32_t pipe_id, uint32_t queue_id,
|
||||
uint32_t __user *wptr, uint32_t wptr_shift,
|
||||
uint32_t wptr_mask, struct mm_struct *mm)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
struct cik_mqd *m;
|
||||
uint32_t *mqd_hqd;
|
||||
uint32_t reg, wptr_val, data;
|
||||
@ -220,7 +204,7 @@ static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
|
||||
|
||||
m = get_mqd(mqd);
|
||||
|
||||
acquire_queue(kgd, pipe_id, queue_id);
|
||||
acquire_queue(adev, pipe_id, queue_id);
|
||||
|
||||
/* HQD registers extend from CP_MQD_BASE_ADDR to CP_MQD_CONTROL. */
|
||||
mqd_hqd = &m->cp_mqd_base_addr_lo;
|
||||
@ -239,25 +223,24 @@ static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
|
||||
* release srbm_mutex to avoid circular dependency between
|
||||
* srbm_mutex->mm_sem->reservation_ww_class_mutex->srbm_mutex.
|
||||
*/
|
||||
release_queue(kgd);
|
||||
release_queue(adev);
|
||||
valid_wptr = read_user_wptr(mm, wptr, wptr_val);
|
||||
acquire_queue(kgd, pipe_id, queue_id);
|
||||
acquire_queue(adev, pipe_id, queue_id);
|
||||
if (valid_wptr)
|
||||
WREG32(mmCP_HQD_PQ_WPTR, (wptr_val << wptr_shift) & wptr_mask);
|
||||
|
||||
data = REG_SET_FIELD(m->cp_hqd_active, CP_HQD_ACTIVE, ACTIVE, 1);
|
||||
WREG32(mmCP_HQD_ACTIVE, data);
|
||||
|
||||
release_queue(kgd);
|
||||
release_queue(adev);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int kgd_hqd_dump(struct kgd_dev *kgd,
|
||||
static int kgd_hqd_dump(struct amdgpu_device *adev,
|
||||
uint32_t pipe_id, uint32_t queue_id,
|
||||
uint32_t (**dump)[2], uint32_t *n_regs)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
uint32_t i = 0, reg;
|
||||
#define HQD_N_REGS (35+4)
|
||||
#define DUMP_REG(addr) do { \
|
||||
@ -271,7 +254,7 @@ static int kgd_hqd_dump(struct kgd_dev *kgd,
|
||||
if (*dump == NULL)
|
||||
return -ENOMEM;
|
||||
|
||||
acquire_queue(kgd, pipe_id, queue_id);
|
||||
acquire_queue(adev, pipe_id, queue_id);
|
||||
|
||||
DUMP_REG(mmCOMPUTE_STATIC_THREAD_MGMT_SE0);
|
||||
DUMP_REG(mmCOMPUTE_STATIC_THREAD_MGMT_SE1);
|
||||
@ -281,7 +264,7 @@ static int kgd_hqd_dump(struct kgd_dev *kgd,
|
||||
for (reg = mmCP_MQD_BASE_ADDR; reg <= mmCP_MQD_CONTROL; reg++)
|
||||
DUMP_REG(reg);
|
||||
|
||||
release_queue(kgd);
|
||||
release_queue(adev);
|
||||
|
||||
WARN_ON_ONCE(i != HQD_N_REGS);
|
||||
*n_regs = i;
|
||||
@ -289,10 +272,9 @@ static int kgd_hqd_dump(struct kgd_dev *kgd,
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd,
|
||||
static int kgd_hqd_sdma_load(struct amdgpu_device *adev, void *mqd,
|
||||
uint32_t __user *wptr, struct mm_struct *mm)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
struct cik_sdma_rlc_registers *m;
|
||||
unsigned long end_jiffies;
|
||||
uint32_t sdma_rlc_reg_offset;
|
||||
@ -345,11 +327,10 @@ static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd,
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int kgd_hqd_sdma_dump(struct kgd_dev *kgd,
|
||||
static int kgd_hqd_sdma_dump(struct amdgpu_device *adev,
|
||||
uint32_t engine_id, uint32_t queue_id,
|
||||
uint32_t (**dump)[2], uint32_t *n_regs)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
uint32_t sdma_offset = engine_id * SDMA1_REGISTER_OFFSET +
|
||||
queue_id * KFD_CIK_SDMA_QUEUE_OFFSET;
|
||||
uint32_t i = 0, reg;
|
||||
@ -372,15 +353,15 @@ static int kgd_hqd_sdma_dump(struct kgd_dev *kgd,
|
||||
return 0;
|
||||
}
|
||||
|
||||
static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address,
|
||||
uint32_t pipe_id, uint32_t queue_id)
|
||||
static bool kgd_hqd_is_occupied(struct amdgpu_device *adev,
|
||||
uint64_t queue_address, uint32_t pipe_id,
|
||||
uint32_t queue_id)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
uint32_t act;
|
||||
bool retval = false;
|
||||
uint32_t low, high;
|
||||
|
||||
acquire_queue(kgd, pipe_id, queue_id);
|
||||
acquire_queue(adev, pipe_id, queue_id);
|
||||
act = RREG32(mmCP_HQD_ACTIVE);
|
||||
if (act) {
|
||||
low = lower_32_bits(queue_address >> 8);
|
||||
@ -390,13 +371,12 @@ static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address,
|
||||
high == RREG32(mmCP_HQD_PQ_BASE_HI))
|
||||
retval = true;
|
||||
}
|
||||
release_queue(kgd);
|
||||
release_queue(adev);
|
||||
return retval;
|
||||
}
|
||||
|
||||
static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd)
|
||||
static bool kgd_hqd_sdma_is_occupied(struct amdgpu_device *adev, void *mqd)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
struct cik_sdma_rlc_registers *m;
|
||||
uint32_t sdma_rlc_reg_offset;
|
||||
uint32_t sdma_rlc_rb_cntl;
|
||||
@ -412,12 +392,11 @@ static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd)
|
||||
return false;
|
||||
}
|
||||
|
||||
static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd,
|
||||
static int kgd_hqd_destroy(struct amdgpu_device *adev, void *mqd,
|
||||
enum kfd_preempt_type reset_type,
|
||||
unsigned int utimeout, uint32_t pipe_id,
|
||||
uint32_t queue_id)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
uint32_t temp;
|
||||
enum hqd_dequeue_request_type type;
|
||||
unsigned long flags, end_jiffies;
|
||||
@ -426,7 +405,7 @@ static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd,
|
||||
if (amdgpu_in_reset(adev))
|
||||
return -EIO;
|
||||
|
||||
acquire_queue(kgd, pipe_id, queue_id);
|
||||
acquire_queue(adev, pipe_id, queue_id);
|
||||
WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, 0);
|
||||
|
||||
switch (reset_type) {
|
||||
@ -504,20 +483,19 @@ loop:
|
||||
break;
|
||||
if (time_after(jiffies, end_jiffies)) {
|
||||
pr_err("cp queue preemption time out\n");
|
||||
release_queue(kgd);
|
||||
release_queue(adev);
|
||||
return -ETIME;
|
||||
}
|
||||
usleep_range(500, 1000);
|
||||
}
|
||||
|
||||
release_queue(kgd);
|
||||
release_queue(adev);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
|
||||
static int kgd_hqd_sdma_destroy(struct amdgpu_device *adev, void *mqd,
|
||||
unsigned int utimeout)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
struct cik_sdma_rlc_registers *m;
|
||||
uint32_t sdma_rlc_reg_offset;
|
||||
uint32_t temp;
|
||||
@ -551,9 +529,8 @@ static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int kgd_address_watch_disable(struct kgd_dev *kgd)
|
||||
static int kgd_address_watch_disable(struct amdgpu_device *adev)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
union TCP_WATCH_CNTL_BITS cntl;
|
||||
unsigned int i;
|
||||
|
||||
@ -571,13 +548,12 @@ static int kgd_address_watch_disable(struct kgd_dev *kgd)
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int kgd_address_watch_execute(struct kgd_dev *kgd,
|
||||
static int kgd_address_watch_execute(struct amdgpu_device *adev,
|
||||
unsigned int watch_point_id,
|
||||
uint32_t cntl_val,
|
||||
uint32_t addr_hi,
|
||||
uint32_t addr_lo)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
union TCP_WATCH_CNTL_BITS cntl;
|
||||
|
||||
cntl.u32All = cntl_val;
|
||||
@ -602,11 +578,10 @@ static int kgd_address_watch_execute(struct kgd_dev *kgd,
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int kgd_wave_control_execute(struct kgd_dev *kgd,
|
||||
static int kgd_wave_control_execute(struct amdgpu_device *adev,
|
||||
uint32_t gfx_index_val,
|
||||
uint32_t sq_cmd)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
uint32_t data;
|
||||
|
||||
mutex_lock(&adev->grbm_idx_mutex);
|
||||
@ -627,18 +602,17 @@ static int kgd_wave_control_execute(struct kgd_dev *kgd,
|
||||
return 0;
|
||||
}
|
||||
|
||||
static uint32_t kgd_address_watch_get_offset(struct kgd_dev *kgd,
|
||||
static uint32_t kgd_address_watch_get_offset(struct amdgpu_device *adev,
|
||||
unsigned int watch_point_id,
|
||||
unsigned int reg_offset)
|
||||
{
|
||||
return watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX + reg_offset];
|
||||
}
|
||||
|
||||
static bool get_atc_vmid_pasid_mapping_info(struct kgd_dev *kgd,
|
||||
static bool get_atc_vmid_pasid_mapping_info(struct amdgpu_device *adev,
|
||||
uint8_t vmid, uint16_t *p_pasid)
|
||||
{
|
||||
uint32_t value;
|
||||
struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
|
||||
|
||||
value = RREG32(mmATC_VMID0_PASID_MAPPING + vmid);
|
||||
*p_pasid = value & ATC_VMID0_PASID_MAPPING__PASID_MASK;
|
||||
@ -646,21 +620,17 @@ static bool get_atc_vmid_pasid_mapping_info(struct kgd_dev *kgd,
|
||||
return !!(value & ATC_VMID0_PASID_MAPPING__VALID_MASK);
|
||||
}
|
||||
|
||||
static void set_scratch_backing_va(struct kgd_dev *kgd,
|
||||
static void set_scratch_backing_va(struct amdgpu_device *adev,
|
||||
uint64_t va, uint32_t vmid)
|
||||
{
|
||||
struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
|
||||
|
||||
lock_srbm(kgd, 0, 0, 0, vmid);
|
||||
lock_srbm(adev, 0, 0, 0, vmid);
|
||||
WREG32(mmSH_HIDDEN_PRIVATE_BASE_VMID, va);
|
||||
unlock_srbm(kgd);
|
||||
unlock_srbm(adev);
|
||||
}
|
||||
|
||||
static void set_vm_context_page_table_base(struct kgd_dev *kgd, uint32_t vmid,
|
||||
uint64_t page_table_base)
|
||||
static void set_vm_context_page_table_base(struct amdgpu_device *adev,
|
||||
uint32_t vmid, uint64_t page_table_base)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
|
||||
if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid)) {
|
||||
pr_err("trying to set page table base for wrong VMID\n");
|
||||
return;
|
||||
@ -676,10 +646,8 @@ static void set_vm_context_page_table_base(struct kgd_dev *kgd, uint32_t vmid,
|
||||
* @vmid: vmid pointer
|
||||
* read vmid from register (CIK).
|
||||
*/
|
||||
static uint32_t read_vmid_from_vmfault_reg(struct kgd_dev *kgd)
|
||||
static uint32_t read_vmid_from_vmfault_reg(struct amdgpu_device *adev)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
|
||||
uint32_t status = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_STATUS);
|
||||
|
||||
return REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS, VMID);
|
||||
|
||||
@ -39,68 +39,54 @@ enum hqd_dequeue_request_type {
|
||||
RESET_WAVES
|
||||
};
|
||||
|
||||
static inline struct amdgpu_device *get_amdgpu_device(struct kgd_dev *kgd)
|
||||
{
|
||||
return (struct amdgpu_device *)kgd;
|
||||
}
|
||||
|
||||
static void lock_srbm(struct kgd_dev *kgd, uint32_t mec, uint32_t pipe,
|
||||
static void lock_srbm(struct amdgpu_device *adev, uint32_t mec, uint32_t pipe,
|
||||
uint32_t queue, uint32_t vmid)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
uint32_t value = PIPEID(pipe) | MEID(mec) | VMID(vmid) | QUEUEID(queue);
|
||||
|
||||
mutex_lock(&adev->srbm_mutex);
|
||||
WREG32(mmSRBM_GFX_CNTL, value);
|
||||
}
|
||||
|
||||
static void unlock_srbm(struct kgd_dev *kgd)
|
||||
static void unlock_srbm(struct amdgpu_device *adev)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
|
||||
WREG32(mmSRBM_GFX_CNTL, 0);
|
||||
mutex_unlock(&adev->srbm_mutex);
|
||||
}
|
||||
|
||||
static void acquire_queue(struct kgd_dev *kgd, uint32_t pipe_id,
|
||||
static void acquire_queue(struct amdgpu_device *adev, uint32_t pipe_id,
|
||||
uint32_t queue_id)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
|
||||
uint32_t mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
|
||||
uint32_t pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec);
|
||||
|
||||
lock_srbm(kgd, mec, pipe, queue_id, 0);
|
||||
lock_srbm(adev, mec, pipe, queue_id, 0);
|
||||
}
|
||||
|
||||
static void release_queue(struct kgd_dev *kgd)
|
||||
static void release_queue(struct amdgpu_device *adev)
|
||||
{
|
||||
unlock_srbm(kgd);
|
||||
unlock_srbm(adev);
|
||||
}
|
||||
|
||||
static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid,
|
||||
static void kgd_program_sh_mem_settings(struct amdgpu_device *adev, uint32_t vmid,
|
||||
uint32_t sh_mem_config,
|
||||
uint32_t sh_mem_ape1_base,
|
||||
uint32_t sh_mem_ape1_limit,
|
||||
uint32_t sh_mem_bases)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
|
||||
lock_srbm(kgd, 0, 0, 0, vmid);
|
||||
lock_srbm(adev, 0, 0, 0, vmid);
|
||||
|
||||
WREG32(mmSH_MEM_CONFIG, sh_mem_config);
|
||||
WREG32(mmSH_MEM_APE1_BASE, sh_mem_ape1_base);
|
||||
WREG32(mmSH_MEM_APE1_LIMIT, sh_mem_ape1_limit);
|
||||
WREG32(mmSH_MEM_BASES, sh_mem_bases);
|
||||
|
||||
unlock_srbm(kgd);
|
||||
unlock_srbm(adev);
|
||||
}
|
||||
|
||||
static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, u32 pasid,
|
||||
static int kgd_set_pasid_vmid_mapping(struct amdgpu_device *adev, u32 pasid,
|
||||
unsigned int vmid)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
|
||||
/*
|
||||
* We have to assume that there is no outstanding mapping.
|
||||
* The ATC_VMID_PASID_MAPPING_UPDATE_STATUS bit could be 0 because
|
||||
@ -123,21 +109,20 @@ static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, u32 pasid,
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id)
|
||||
static int kgd_init_interrupts(struct amdgpu_device *adev, uint32_t pipe_id)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
uint32_t mec;
|
||||
uint32_t pipe;
|
||||
|
||||
mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
|
||||
pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec);
|
||||
|
||||
lock_srbm(kgd, mec, pipe, 0, 0);
|
||||
lock_srbm(adev, mec, pipe, 0, 0);
|
||||
|
||||
WREG32(mmCPC_INT_CNTL, CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK |
|
||||
CP_INT_CNTL_RING0__OPCODE_ERROR_INT_ENABLE_MASK);
|
||||
|
||||
unlock_srbm(kgd);
|
||||
unlock_srbm(adev);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@ -165,12 +150,11 @@ static inline struct vi_sdma_mqd *get_sdma_mqd(void *mqd)
|
||||
return (struct vi_sdma_mqd *)mqd;
|
||||
}
|
||||
|
||||
static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
|
||||
uint32_t queue_id, uint32_t __user *wptr,
|
||||
uint32_t wptr_shift, uint32_t wptr_mask,
|
||||
struct mm_struct *mm)
|
||||
static int kgd_hqd_load(struct amdgpu_device *adev, void *mqd,
|
||||
uint32_t pipe_id, uint32_t queue_id,
|
||||
uint32_t __user *wptr, uint32_t wptr_shift,
|
||||
uint32_t wptr_mask, struct mm_struct *mm)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
struct vi_mqd *m;
|
||||
uint32_t *mqd_hqd;
|
||||
uint32_t reg, wptr_val, data;
|
||||
@ -178,7 +162,7 @@ static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
|
||||
|
||||
m = get_mqd(mqd);
|
||||
|
||||
acquire_queue(kgd, pipe_id, queue_id);
|
||||
acquire_queue(adev, pipe_id, queue_id);
|
||||
|
||||
/* HIQ is set during driver init period with vmid set to 0*/
|
||||
if (m->cp_hqd_vmid == 0) {
|
||||
@ -206,7 +190,7 @@ static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
|
||||
* on ASICs that do not support context-save.
|
||||
* EOP writes/reads can start anywhere in the ring.
|
||||
*/
|
||||
if (get_amdgpu_device(kgd)->asic_type != CHIP_TONGA) {
|
||||
if (adev->asic_type != CHIP_TONGA) {
|
||||
WREG32(mmCP_HQD_EOP_RPTR, m->cp_hqd_eop_rptr);
|
||||
WREG32(mmCP_HQD_EOP_WPTR, m->cp_hqd_eop_wptr);
|
||||
WREG32(mmCP_HQD_EOP_WPTR_MEM, m->cp_hqd_eop_wptr_mem);
|
||||
@ -226,25 +210,24 @@ static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
|
||||
* release srbm_mutex to avoid circular dependency between
|
||||
* srbm_mutex->mm_sem->reservation_ww_class_mutex->srbm_mutex.
|
||||
*/
|
||||
release_queue(kgd);
|
||||
release_queue(adev);
|
||||
valid_wptr = read_user_wptr(mm, wptr, wptr_val);
|
||||
acquire_queue(kgd, pipe_id, queue_id);
|
||||
acquire_queue(adev, pipe_id, queue_id);
|
||||
if (valid_wptr)
|
||||
WREG32(mmCP_HQD_PQ_WPTR, (wptr_val << wptr_shift) & wptr_mask);
|
||||
|
||||
data = REG_SET_FIELD(m->cp_hqd_active, CP_HQD_ACTIVE, ACTIVE, 1);
|
||||
WREG32(mmCP_HQD_ACTIVE, data);
|
||||
|
||||
release_queue(kgd);
|
||||
release_queue(adev);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int kgd_hqd_dump(struct kgd_dev *kgd,
|
||||
static int kgd_hqd_dump(struct amdgpu_device *adev,
|
||||
uint32_t pipe_id, uint32_t queue_id,
|
||||
uint32_t (**dump)[2], uint32_t *n_regs)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
uint32_t i = 0, reg;
|
||||
#define HQD_N_REGS (54+4)
|
||||
#define DUMP_REG(addr) do { \
|
||||
@ -258,7 +241,7 @@ static int kgd_hqd_dump(struct kgd_dev *kgd,
|
||||
if (*dump == NULL)
|
||||
return -ENOMEM;
|
||||
|
||||
acquire_queue(kgd, pipe_id, queue_id);
|
||||
acquire_queue(adev, pipe_id, queue_id);
|
||||
|
||||
DUMP_REG(mmCOMPUTE_STATIC_THREAD_MGMT_SE0);
|
||||
DUMP_REG(mmCOMPUTE_STATIC_THREAD_MGMT_SE1);
|
||||
@ -268,7 +251,7 @@ static int kgd_hqd_dump(struct kgd_dev *kgd,
|
||||
for (reg = mmCP_MQD_BASE_ADDR; reg <= mmCP_HQD_EOP_DONES; reg++)
|
||||
DUMP_REG(reg);
|
||||
|
||||
release_queue(kgd);
|
||||
release_queue(adev);
|
||||
|
||||
WARN_ON_ONCE(i != HQD_N_REGS);
|
||||
*n_regs = i;
|
||||
@ -276,10 +259,9 @@ static int kgd_hqd_dump(struct kgd_dev *kgd,
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd,
|
||||
static int kgd_hqd_sdma_load(struct amdgpu_device *adev, void *mqd,
|
||||
uint32_t __user *wptr, struct mm_struct *mm)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
struct vi_sdma_mqd *m;
|
||||
unsigned long end_jiffies;
|
||||
uint32_t sdma_rlc_reg_offset;
|
||||
@ -331,11 +313,10 @@ static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd,
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int kgd_hqd_sdma_dump(struct kgd_dev *kgd,
|
||||
static int kgd_hqd_sdma_dump(struct amdgpu_device *adev,
|
||||
uint32_t engine_id, uint32_t queue_id,
|
||||
uint32_t (**dump)[2], uint32_t *n_regs)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
uint32_t sdma_offset = engine_id * SDMA1_REGISTER_OFFSET +
|
||||
queue_id * KFD_VI_SDMA_QUEUE_OFFSET;
|
||||
uint32_t i = 0, reg;
|
||||
@ -367,15 +348,15 @@ static int kgd_hqd_sdma_dump(struct kgd_dev *kgd,
|
||||
return 0;
|
||||
}
|
||||
|
||||
static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address,
|
||||
uint32_t pipe_id, uint32_t queue_id)
|
||||
static bool kgd_hqd_is_occupied(struct amdgpu_device *adev,
|
||||
uint64_t queue_address, uint32_t pipe_id,
|
||||
uint32_t queue_id)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
uint32_t act;
|
||||
bool retval = false;
|
||||
uint32_t low, high;
|
||||
|
||||
acquire_queue(kgd, pipe_id, queue_id);
|
||||
acquire_queue(adev, pipe_id, queue_id);
|
||||
act = RREG32(mmCP_HQD_ACTIVE);
|
||||
if (act) {
|
||||
low = lower_32_bits(queue_address >> 8);
|
||||
@ -385,13 +366,12 @@ static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address,
|
||||
high == RREG32(mmCP_HQD_PQ_BASE_HI))
|
||||
retval = true;
|
||||
}
|
||||
release_queue(kgd);
|
||||
release_queue(adev);
|
||||
return retval;
|
||||
}
|
||||
|
||||
static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd)
|
||||
static bool kgd_hqd_sdma_is_occupied(struct amdgpu_device *adev, void *mqd)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
struct vi_sdma_mqd *m;
|
||||
uint32_t sdma_rlc_reg_offset;
|
||||
uint32_t sdma_rlc_rb_cntl;
|
||||
@ -407,12 +387,11 @@ static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd)
|
||||
return false;
|
||||
}
|
||||
|
||||
static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd,
|
||||
static int kgd_hqd_destroy(struct amdgpu_device *adev, void *mqd,
|
||||
enum kfd_preempt_type reset_type,
|
||||
unsigned int utimeout, uint32_t pipe_id,
|
||||
uint32_t queue_id)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
uint32_t temp;
|
||||
enum hqd_dequeue_request_type type;
|
||||
unsigned long flags, end_jiffies;
|
||||
@ -422,7 +401,7 @@ static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd,
|
||||
if (amdgpu_in_reset(adev))
|
||||
return -EIO;
|
||||
|
||||
acquire_queue(kgd, pipe_id, queue_id);
|
||||
acquire_queue(adev, pipe_id, queue_id);
|
||||
|
||||
if (m->cp_hqd_vmid == 0)
|
||||
WREG32_FIELD(RLC_CP_SCHEDULERS, scheduler1, 0);
|
||||
@ -502,20 +481,19 @@ loop:
|
||||
break;
|
||||
if (time_after(jiffies, end_jiffies)) {
|
||||
pr_err("cp queue preemption time out.\n");
|
||||
release_queue(kgd);
|
||||
release_queue(adev);
|
||||
return -ETIME;
|
||||
}
|
||||
usleep_range(500, 1000);
|
||||
}
|
||||
|
||||
release_queue(kgd);
|
||||
release_queue(adev);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
|
||||
static int kgd_hqd_sdma_destroy(struct amdgpu_device *adev, void *mqd,
|
||||
unsigned int utimeout)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
struct vi_sdma_mqd *m;
|
||||
uint32_t sdma_rlc_reg_offset;
|
||||
uint32_t temp;
|
||||
@ -549,11 +527,10 @@ static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
|
||||
return 0;
|
||||
}
|
||||
|
||||
static bool get_atc_vmid_pasid_mapping_info(struct kgd_dev *kgd,
|
||||
static bool get_atc_vmid_pasid_mapping_info(struct amdgpu_device *adev,
|
||||
uint8_t vmid, uint16_t *p_pasid)
|
||||
{
|
||||
uint32_t value;
|
||||
struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
|
||||
|
||||
value = RREG32(mmATC_VMID0_PASID_MAPPING + vmid);
|
||||
*p_pasid = value & ATC_VMID0_PASID_MAPPING__PASID_MASK;
|
||||
@ -561,12 +538,12 @@ static bool get_atc_vmid_pasid_mapping_info(struct kgd_dev *kgd,
|
||||
return !!(value & ATC_VMID0_PASID_MAPPING__VALID_MASK);
|
||||
}
|
||||
|
||||
static int kgd_address_watch_disable(struct kgd_dev *kgd)
|
||||
static int kgd_address_watch_disable(struct amdgpu_device *adev)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int kgd_address_watch_execute(struct kgd_dev *kgd,
|
||||
static int kgd_address_watch_execute(struct amdgpu_device *adev,
|
||||
unsigned int watch_point_id,
|
||||
uint32_t cntl_val,
|
||||
uint32_t addr_hi,
|
||||
@ -575,11 +552,10 @@ static int kgd_address_watch_execute(struct kgd_dev *kgd,
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int kgd_wave_control_execute(struct kgd_dev *kgd,
|
||||
static int kgd_wave_control_execute(struct amdgpu_device *adev,
|
||||
uint32_t gfx_index_val,
|
||||
uint32_t sq_cmd)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
uint32_t data = 0;
|
||||
|
||||
mutex_lock(&adev->grbm_idx_mutex);
|
||||
@ -600,28 +576,24 @@ static int kgd_wave_control_execute(struct kgd_dev *kgd,
|
||||
return 0;
|
||||
}
|
||||
|
||||
static uint32_t kgd_address_watch_get_offset(struct kgd_dev *kgd,
|
||||
static uint32_t kgd_address_watch_get_offset(struct amdgpu_device *adev,
|
||||
unsigned int watch_point_id,
|
||||
unsigned int reg_offset)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void set_scratch_backing_va(struct kgd_dev *kgd,
|
||||
static void set_scratch_backing_va(struct amdgpu_device *adev,
|
||||
uint64_t va, uint32_t vmid)
|
||||
{
|
||||
struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
|
||||
|
||||
lock_srbm(kgd, 0, 0, 0, vmid);
|
||||
lock_srbm(adev, 0, 0, 0, vmid);
|
||||
WREG32(mmSH_HIDDEN_PRIVATE_BASE_VMID, va);
|
||||
unlock_srbm(kgd);
|
||||
unlock_srbm(adev);
|
||||
}
|
||||
|
||||
static void set_vm_context_page_table_base(struct kgd_dev *kgd, uint32_t vmid,
|
||||
uint64_t page_table_base)
|
||||
static void set_vm_context_page_table_base(struct amdgpu_device *adev,
|
||||
uint32_t vmid, uint64_t page_table_base)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
|
||||
if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid)) {
|
||||
pr_err("trying to set page table base for wrong VMID\n");
|
||||
return;
|
||||
|
||||
@ -46,37 +46,26 @@ enum hqd_dequeue_request_type {
|
||||
SAVE_WAVES
|
||||
};
|
||||
|
||||
static inline struct amdgpu_device *get_amdgpu_device(struct kgd_dev *kgd)
|
||||
{
|
||||
return (struct amdgpu_device *)kgd;
|
||||
}
|
||||
|
||||
static void lock_srbm(struct kgd_dev *kgd, uint32_t mec, uint32_t pipe,
|
||||
static void lock_srbm(struct amdgpu_device *adev, uint32_t mec, uint32_t pipe,
|
||||
uint32_t queue, uint32_t vmid)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
|
||||
mutex_lock(&adev->srbm_mutex);
|
||||
soc15_grbm_select(adev, mec, pipe, queue, vmid);
|
||||
}
|
||||
|
||||
static void unlock_srbm(struct kgd_dev *kgd)
|
||||
static void unlock_srbm(struct amdgpu_device *adev)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
|
||||
soc15_grbm_select(adev, 0, 0, 0, 0);
|
||||
mutex_unlock(&adev->srbm_mutex);
|
||||
}
|
||||
|
||||
static void acquire_queue(struct kgd_dev *kgd, uint32_t pipe_id,
|
||||
static void acquire_queue(struct amdgpu_device *adev, uint32_t pipe_id,
|
||||
uint32_t queue_id)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
|
||||
uint32_t mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
|
||||
uint32_t pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec);
|
||||
|
||||
lock_srbm(kgd, mec, pipe, queue_id, 0);
|
||||
lock_srbm(adev, mec, pipe, queue_id, 0);
|
||||
}
|
||||
|
||||
static uint64_t get_queue_mask(struct amdgpu_device *adev,
|
||||
@ -88,33 +77,29 @@ static uint64_t get_queue_mask(struct amdgpu_device *adev,
|
||||
return 1ull << bit;
|
||||
}
|
||||
|
||||
static void release_queue(struct kgd_dev *kgd)
|
||||
static void release_queue(struct amdgpu_device *adev)
|
||||
{
|
||||
unlock_srbm(kgd);
|
||||
unlock_srbm(adev);
|
||||
}
|
||||
|
||||
void kgd_gfx_v9_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid,
|
||||
void kgd_gfx_v9_program_sh_mem_settings(struct amdgpu_device *adev, uint32_t vmid,
|
||||
uint32_t sh_mem_config,
|
||||
uint32_t sh_mem_ape1_base,
|
||||
uint32_t sh_mem_ape1_limit,
|
||||
uint32_t sh_mem_bases)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
|
||||
lock_srbm(kgd, 0, 0, 0, vmid);
|
||||
lock_srbm(adev, 0, 0, 0, vmid);
|
||||
|
||||
WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmSH_MEM_CONFIG), sh_mem_config);
|
||||
WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmSH_MEM_BASES), sh_mem_bases);
|
||||
/* APE1 no longer exists on GFX9 */
|
||||
|
||||
unlock_srbm(kgd);
|
||||
unlock_srbm(adev);
|
||||
}
|
||||
|
||||
int kgd_gfx_v9_set_pasid_vmid_mapping(struct kgd_dev *kgd, u32 pasid,
|
||||
int kgd_gfx_v9_set_pasid_vmid_mapping(struct amdgpu_device *adev, u32 pasid,
|
||||
unsigned int vmid)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
|
||||
/*
|
||||
* We have to assume that there is no outstanding mapping.
|
||||
* The ATC_VMID_PASID_MAPPING_UPDATE_STATUS bit could be 0 because
|
||||
@ -171,22 +156,21 @@ int kgd_gfx_v9_set_pasid_vmid_mapping(struct kgd_dev *kgd, u32 pasid,
|
||||
* but still works
|
||||
*/
|
||||
|
||||
int kgd_gfx_v9_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id)
|
||||
int kgd_gfx_v9_init_interrupts(struct amdgpu_device *adev, uint32_t pipe_id)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
uint32_t mec;
|
||||
uint32_t pipe;
|
||||
|
||||
mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
|
||||
pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec);
|
||||
|
||||
lock_srbm(kgd, mec, pipe, 0, 0);
|
||||
lock_srbm(adev, mec, pipe, 0, 0);
|
||||
|
||||
WREG32(SOC15_REG_OFFSET(GC, 0, mmCPC_INT_CNTL),
|
||||
CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK |
|
||||
CP_INT_CNTL_RING0__OPCODE_ERROR_INT_ENABLE_MASK);
|
||||
|
||||
unlock_srbm(kgd);
|
||||
unlock_srbm(adev);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@ -233,19 +217,18 @@ static inline struct v9_sdma_mqd *get_sdma_mqd(void *mqd)
|
||||
return (struct v9_sdma_mqd *)mqd;
|
||||
}
|
||||
|
||||
int kgd_gfx_v9_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
|
||||
uint32_t queue_id, uint32_t __user *wptr,
|
||||
uint32_t wptr_shift, uint32_t wptr_mask,
|
||||
struct mm_struct *mm)
|
||||
int kgd_gfx_v9_hqd_load(struct amdgpu_device *adev, void *mqd,
|
||||
uint32_t pipe_id, uint32_t queue_id,
|
||||
uint32_t __user *wptr, uint32_t wptr_shift,
|
||||
uint32_t wptr_mask, struct mm_struct *mm)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
struct v9_mqd *m;
|
||||
uint32_t *mqd_hqd;
|
||||
uint32_t reg, hqd_base, data;
|
||||
|
||||
m = get_mqd(mqd);
|
||||
|
||||
acquire_queue(kgd, pipe_id, queue_id);
|
||||
acquire_queue(adev, pipe_id, queue_id);
|
||||
|
||||
/* HQD registers extend from CP_MQD_BASE_ADDR to CP_HQD_EOP_WPTR_MEM. */
|
||||
mqd_hqd = &m->cp_mqd_base_addr_lo;
|
||||
@ -308,16 +291,15 @@ int kgd_gfx_v9_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
|
||||
data = REG_SET_FIELD(m->cp_hqd_active, CP_HQD_ACTIVE, ACTIVE, 1);
|
||||
WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_ACTIVE), data);
|
||||
|
||||
release_queue(kgd);
|
||||
release_queue(adev);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int kgd_gfx_v9_hiq_mqd_load(struct kgd_dev *kgd, void *mqd,
|
||||
int kgd_gfx_v9_hiq_mqd_load(struct amdgpu_device *adev, void *mqd,
|
||||
uint32_t pipe_id, uint32_t queue_id,
|
||||
uint32_t doorbell_off)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring;
|
||||
struct v9_mqd *m;
|
||||
uint32_t mec, pipe;
|
||||
@ -325,7 +307,7 @@ int kgd_gfx_v9_hiq_mqd_load(struct kgd_dev *kgd, void *mqd,
|
||||
|
||||
m = get_mqd(mqd);
|
||||
|
||||
acquire_queue(kgd, pipe_id, queue_id);
|
||||
acquire_queue(adev, pipe_id, queue_id);
|
||||
|
||||
mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
|
||||
pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec);
|
||||
@ -361,16 +343,15 @@ int kgd_gfx_v9_hiq_mqd_load(struct kgd_dev *kgd, void *mqd,
|
||||
|
||||
out_unlock:
|
||||
spin_unlock(&adev->gfx.kiq.ring_lock);
|
||||
release_queue(kgd);
|
||||
release_queue(adev);
|
||||
|
||||
return r;
|
||||
}
|
||||
|
||||
int kgd_gfx_v9_hqd_dump(struct kgd_dev *kgd,
|
||||
int kgd_gfx_v9_hqd_dump(struct amdgpu_device *adev,
|
||||
uint32_t pipe_id, uint32_t queue_id,
|
||||
uint32_t (**dump)[2], uint32_t *n_regs)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
uint32_t i = 0, reg;
|
||||
#define HQD_N_REGS 56
|
||||
#define DUMP_REG(addr) do { \
|
||||
@ -384,13 +365,13 @@ int kgd_gfx_v9_hqd_dump(struct kgd_dev *kgd,
|
||||
if (*dump == NULL)
|
||||
return -ENOMEM;
|
||||
|
||||
acquire_queue(kgd, pipe_id, queue_id);
|
||||
acquire_queue(adev, pipe_id, queue_id);
|
||||
|
||||
for (reg = SOC15_REG_OFFSET(GC, 0, mmCP_MQD_BASE_ADDR);
|
||||
reg <= SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_HI); reg++)
|
||||
DUMP_REG(reg);
|
||||
|
||||
release_queue(kgd);
|
||||
release_queue(adev);
|
||||
|
||||
WARN_ON_ONCE(i != HQD_N_REGS);
|
||||
*n_regs = i;
|
||||
@ -398,10 +379,9 @@ int kgd_gfx_v9_hqd_dump(struct kgd_dev *kgd,
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd,
|
||||
static int kgd_hqd_sdma_load(struct amdgpu_device *adev, void *mqd,
|
||||
uint32_t __user *wptr, struct mm_struct *mm)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
struct v9_sdma_mqd *m;
|
||||
uint32_t sdma_rlc_reg_offset;
|
||||
unsigned long end_jiffies;
|
||||
@ -468,11 +448,10 @@ static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd,
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int kgd_hqd_sdma_dump(struct kgd_dev *kgd,
|
||||
static int kgd_hqd_sdma_dump(struct amdgpu_device *adev,
|
||||
uint32_t engine_id, uint32_t queue_id,
|
||||
uint32_t (**dump)[2], uint32_t *n_regs)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
uint32_t sdma_rlc_reg_offset = get_sdma_rlc_reg_offset(adev,
|
||||
engine_id, queue_id);
|
||||
uint32_t i = 0, reg;
|
||||
@ -500,15 +479,15 @@ static int kgd_hqd_sdma_dump(struct kgd_dev *kgd,
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool kgd_gfx_v9_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address,
|
||||
uint32_t pipe_id, uint32_t queue_id)
|
||||
bool kgd_gfx_v9_hqd_is_occupied(struct amdgpu_device *adev,
|
||||
uint64_t queue_address, uint32_t pipe_id,
|
||||
uint32_t queue_id)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
uint32_t act;
|
||||
bool retval = false;
|
||||
uint32_t low, high;
|
||||
|
||||
acquire_queue(kgd, pipe_id, queue_id);
|
||||
acquire_queue(adev, pipe_id, queue_id);
|
||||
act = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_ACTIVE));
|
||||
if (act) {
|
||||
low = lower_32_bits(queue_address >> 8);
|
||||
@ -518,13 +497,12 @@ bool kgd_gfx_v9_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address,
|
||||
high == RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_BASE_HI)))
|
||||
retval = true;
|
||||
}
|
||||
release_queue(kgd);
|
||||
release_queue(adev);
|
||||
return retval;
|
||||
}
|
||||
|
||||
static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd)
|
||||
static bool kgd_hqd_sdma_is_occupied(struct amdgpu_device *adev, void *mqd)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
struct v9_sdma_mqd *m;
|
||||
uint32_t sdma_rlc_reg_offset;
|
||||
uint32_t sdma_rlc_rb_cntl;
|
||||
@ -541,12 +519,11 @@ static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd)
|
||||
return false;
|
||||
}
|
||||
|
||||
int kgd_gfx_v9_hqd_destroy(struct kgd_dev *kgd, void *mqd,
|
||||
int kgd_gfx_v9_hqd_destroy(struct amdgpu_device *adev, void *mqd,
|
||||
enum kfd_preempt_type reset_type,
|
||||
unsigned int utimeout, uint32_t pipe_id,
|
||||
uint32_t queue_id)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
enum hqd_dequeue_request_type type;
|
||||
unsigned long end_jiffies;
|
||||
uint32_t temp;
|
||||
@ -555,7 +532,7 @@ int kgd_gfx_v9_hqd_destroy(struct kgd_dev *kgd, void *mqd,
|
||||
if (amdgpu_in_reset(adev))
|
||||
return -EIO;
|
||||
|
||||
acquire_queue(kgd, pipe_id, queue_id);
|
||||
acquire_queue(adev, pipe_id, queue_id);
|
||||
|
||||
if (m->cp_hqd_vmid == 0)
|
||||
WREG32_FIELD15_RLC(GC, 0, RLC_CP_SCHEDULERS, scheduler1, 0);
|
||||
@ -584,20 +561,19 @@ int kgd_gfx_v9_hqd_destroy(struct kgd_dev *kgd, void *mqd,
|
||||
break;
|
||||
if (time_after(jiffies, end_jiffies)) {
|
||||
pr_err("cp queue preemption time out.\n");
|
||||
release_queue(kgd);
|
||||
release_queue(adev);
|
||||
return -ETIME;
|
||||
}
|
||||
usleep_range(500, 1000);
|
||||
}
|
||||
|
||||
release_queue(kgd);
|
||||
release_queue(adev);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
|
||||
static int kgd_hqd_sdma_destroy(struct amdgpu_device *adev, void *mqd,
|
||||
unsigned int utimeout)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
struct v9_sdma_mqd *m;
|
||||
uint32_t sdma_rlc_reg_offset;
|
||||
uint32_t temp;
|
||||
@ -634,11 +610,10 @@ static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool kgd_gfx_v9_get_atc_vmid_pasid_mapping_info(struct kgd_dev *kgd,
|
||||
bool kgd_gfx_v9_get_atc_vmid_pasid_mapping_info(struct amdgpu_device *adev,
|
||||
uint8_t vmid, uint16_t *p_pasid)
|
||||
{
|
||||
uint32_t value;
|
||||
struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
|
||||
|
||||
value = RREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING)
|
||||
+ vmid);
|
||||
@ -647,12 +622,12 @@ bool kgd_gfx_v9_get_atc_vmid_pasid_mapping_info(struct kgd_dev *kgd,
|
||||
return !!(value & ATC_VMID0_PASID_MAPPING__VALID_MASK);
|
||||
}
|
||||
|
||||
int kgd_gfx_v9_address_watch_disable(struct kgd_dev *kgd)
|
||||
int kgd_gfx_v9_address_watch_disable(struct amdgpu_device *adev)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
int kgd_gfx_v9_address_watch_execute(struct kgd_dev *kgd,
|
||||
int kgd_gfx_v9_address_watch_execute(struct amdgpu_device *adev,
|
||||
unsigned int watch_point_id,
|
||||
uint32_t cntl_val,
|
||||
uint32_t addr_hi,
|
||||
@ -661,11 +636,10 @@ int kgd_gfx_v9_address_watch_execute(struct kgd_dev *kgd,
|
||||
return 0;
|
||||
}
|
||||
|
||||
int kgd_gfx_v9_wave_control_execute(struct kgd_dev *kgd,
|
||||
int kgd_gfx_v9_wave_control_execute(struct amdgpu_device *adev,
|
||||
uint32_t gfx_index_val,
|
||||
uint32_t sq_cmd)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
uint32_t data = 0;
|
||||
|
||||
mutex_lock(&adev->grbm_idx_mutex);
|
||||
@ -686,18 +660,16 @@ int kgd_gfx_v9_wave_control_execute(struct kgd_dev *kgd,
|
||||
return 0;
|
||||
}
|
||||
|
||||
uint32_t kgd_gfx_v9_address_watch_get_offset(struct kgd_dev *kgd,
|
||||
uint32_t kgd_gfx_v9_address_watch_get_offset(struct amdgpu_device *adev,
|
||||
unsigned int watch_point_id,
|
||||
unsigned int reg_offset)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
void kgd_gfx_v9_set_vm_context_page_table_base(struct kgd_dev *kgd,
|
||||
void kgd_gfx_v9_set_vm_context_page_table_base(struct amdgpu_device *adev,
|
||||
uint32_t vmid, uint64_t page_table_base)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
|
||||
if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid)) {
|
||||
pr_err("trying to set page table base for wrong VMID %u\n",
|
||||
vmid);
|
||||
@ -804,7 +776,7 @@ static void get_wave_count(struct amdgpu_device *adev, int queue_idx,
|
||||
*
|
||||
* Reading registers referenced above involves programming GRBM appropriately
|
||||
*/
|
||||
void kgd_gfx_v9_get_cu_occupancy(struct kgd_dev *kgd, int pasid,
|
||||
void kgd_gfx_v9_get_cu_occupancy(struct amdgpu_device *adev, int pasid,
|
||||
int *pasid_wave_cnt, int *max_waves_per_cu)
|
||||
{
|
||||
int qidx;
|
||||
@ -818,10 +790,8 @@ void kgd_gfx_v9_get_cu_occupancy(struct kgd_dev *kgd, int pasid,
|
||||
int pasid_tmp;
|
||||
int max_queue_cnt;
|
||||
int vmid_wave_cnt = 0;
|
||||
struct amdgpu_device *adev;
|
||||
DECLARE_BITMAP(cp_queue_bitmap, KGD_MAX_QUEUES);
|
||||
|
||||
adev = get_amdgpu_device(kgd);
|
||||
lock_spi_csq_mutexes(adev);
|
||||
soc15_grbm_select(adev, 1, 0, 0, 0);
|
||||
|
||||
@ -882,12 +852,10 @@ void kgd_gfx_v9_get_cu_occupancy(struct kgd_dev *kgd, int pasid,
|
||||
adev->gfx.cu_info.max_waves_per_simd;
|
||||
}
|
||||
|
||||
void kgd_gfx_v9_program_trap_handler_settings(struct kgd_dev *kgd,
|
||||
void kgd_gfx_v9_program_trap_handler_settings(struct amdgpu_device *adev,
|
||||
uint32_t vmid, uint64_t tba_addr, uint64_t tma_addr)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
|
||||
lock_srbm(kgd, 0, 0, 0, vmid);
|
||||
lock_srbm(adev, 0, 0, 0, vmid);
|
||||
|
||||
/*
|
||||
* Program TBA registers
|
||||
@ -905,7 +873,7 @@ void kgd_gfx_v9_program_trap_handler_settings(struct kgd_dev *kgd,
|
||||
WREG32(SOC15_REG_OFFSET(GC, 0, mmSQ_SHADER_TMA_HI),
|
||||
upper_32_bits(tma_addr >> 8));
|
||||
|
||||
unlock_srbm(kgd);
|
||||
unlock_srbm(adev);
|
||||
}
|
||||
|
||||
const struct kfd2kgd_calls gfx_v9_kfd2kgd = {
|
||||
|
||||
@ -22,48 +22,49 @@
|
||||
|
||||
|
||||
|
||||
void kgd_gfx_v9_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid,
|
||||
void kgd_gfx_v9_program_sh_mem_settings(struct amdgpu_device *adev, uint32_t vmid,
|
||||
uint32_t sh_mem_config,
|
||||
uint32_t sh_mem_ape1_base, uint32_t sh_mem_ape1_limit,
|
||||
uint32_t sh_mem_bases);
|
||||
int kgd_gfx_v9_set_pasid_vmid_mapping(struct kgd_dev *kgd, u32 pasid,
|
||||
int kgd_gfx_v9_set_pasid_vmid_mapping(struct amdgpu_device *adev, u32 pasid,
|
||||
unsigned int vmid);
|
||||
int kgd_gfx_v9_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id);
|
||||
int kgd_gfx_v9_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
|
||||
int kgd_gfx_v9_init_interrupts(struct amdgpu_device *adev, uint32_t pipe_id);
|
||||
int kgd_gfx_v9_hqd_load(struct amdgpu_device *adev, void *mqd, uint32_t pipe_id,
|
||||
uint32_t queue_id, uint32_t __user *wptr,
|
||||
uint32_t wptr_shift, uint32_t wptr_mask,
|
||||
struct mm_struct *mm);
|
||||
int kgd_gfx_v9_hiq_mqd_load(struct kgd_dev *kgd, void *mqd,
|
||||
int kgd_gfx_v9_hiq_mqd_load(struct amdgpu_device *adev, void *mqd,
|
||||
uint32_t pipe_id, uint32_t queue_id,
|
||||
uint32_t doorbell_off);
|
||||
int kgd_gfx_v9_hqd_dump(struct kgd_dev *kgd,
|
||||
int kgd_gfx_v9_hqd_dump(struct amdgpu_device *adev,
|
||||
uint32_t pipe_id, uint32_t queue_id,
|
||||
uint32_t (**dump)[2], uint32_t *n_regs);
|
||||
bool kgd_gfx_v9_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address,
|
||||
uint32_t pipe_id, uint32_t queue_id);
|
||||
int kgd_gfx_v9_hqd_destroy(struct kgd_dev *kgd, void *mqd,
|
||||
bool kgd_gfx_v9_hqd_is_occupied(struct amdgpu_device *adev,
|
||||
uint64_t queue_address, uint32_t pipe_id,
|
||||
uint32_t queue_id);
|
||||
int kgd_gfx_v9_hqd_destroy(struct amdgpu_device *adev, void *mqd,
|
||||
enum kfd_preempt_type reset_type,
|
||||
unsigned int utimeout, uint32_t pipe_id,
|
||||
uint32_t queue_id);
|
||||
int kgd_gfx_v9_address_watch_disable(struct kgd_dev *kgd);
|
||||
int kgd_gfx_v9_address_watch_execute(struct kgd_dev *kgd,
|
||||
int kgd_gfx_v9_address_watch_disable(struct amdgpu_device *adev);
|
||||
int kgd_gfx_v9_address_watch_execute(struct amdgpu_device *adev,
|
||||
unsigned int watch_point_id,
|
||||
uint32_t cntl_val,
|
||||
uint32_t addr_hi,
|
||||
uint32_t addr_lo);
|
||||
int kgd_gfx_v9_wave_control_execute(struct kgd_dev *kgd,
|
||||
int kgd_gfx_v9_wave_control_execute(struct amdgpu_device *adev,
|
||||
uint32_t gfx_index_val,
|
||||
uint32_t sq_cmd);
|
||||
uint32_t kgd_gfx_v9_address_watch_get_offset(struct kgd_dev *kgd,
|
||||
uint32_t kgd_gfx_v9_address_watch_get_offset(struct amdgpu_device *adev,
|
||||
unsigned int watch_point_id,
|
||||
unsigned int reg_offset);
|
||||
|
||||
bool kgd_gfx_v9_get_atc_vmid_pasid_mapping_info(struct kgd_dev *kgd,
|
||||
bool kgd_gfx_v9_get_atc_vmid_pasid_mapping_info(struct amdgpu_device *adev,
|
||||
uint8_t vmid, uint16_t *p_pasid);
|
||||
|
||||
void kgd_gfx_v9_set_vm_context_page_table_base(struct kgd_dev *kgd,
|
||||
void kgd_gfx_v9_set_vm_context_page_table_base(struct amdgpu_device *adev,
|
||||
uint32_t vmid, uint64_t page_table_base);
|
||||
void kgd_gfx_v9_get_cu_occupancy(struct kgd_dev *kgd, int pasid,
|
||||
void kgd_gfx_v9_get_cu_occupancy(struct amdgpu_device *adev, int pasid,
|
||||
int *pasid_wave_cnt, int *max_waves_per_cu);
|
||||
void kgd_gfx_v9_program_trap_handler_settings(struct kgd_dev *kgd,
|
||||
void kgd_gfx_v9_program_trap_handler_settings(struct amdgpu_device *adev,
|
||||
uint32_t vmid, uint64_t tba_addr, uint64_t tma_addr);
|
||||
|
||||
@ -60,12 +60,6 @@ static const char * const domain_bit_to_string[] = {
|
||||
|
||||
static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work);
|
||||
|
||||
|
||||
static inline struct amdgpu_device *get_amdgpu_device(struct kgd_dev *kgd)
|
||||
{
|
||||
return (struct amdgpu_device *)kgd;
|
||||
}
|
||||
|
||||
static bool kfd_mem_is_attached(struct amdgpu_vm *avm,
|
||||
struct kgd_mem *mem)
|
||||
{
|
||||
@ -126,8 +120,19 @@ static size_t amdgpu_amdkfd_acc_size(uint64_t size)
|
||||
PAGE_ALIGN(size);
|
||||
}
|
||||
|
||||
/**
|
||||
* @amdgpu_amdkfd_reserve_mem_limit() - Decrease available memory by size
|
||||
* of buffer including any reserved for control structures
|
||||
*
|
||||
* @adev: Device to which allocated BO belongs to
|
||||
* @size: Size of buffer, in bytes, encapsulated by B0. This should be
|
||||
* equivalent to amdgpu_bo_size(BO)
|
||||
* @alloc_flag: Flag used in allocating a BO as noted above
|
||||
*
|
||||
* Return: returns -ENOMEM in case of error, ZERO otherwise
|
||||
*/
|
||||
static int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev,
|
||||
uint64_t size, u32 domain, bool sg)
|
||||
uint64_t size, u32 alloc_flag)
|
||||
{
|
||||
uint64_t reserved_for_pt =
|
||||
ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size);
|
||||
@ -137,20 +142,24 @@ static int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev,
|
||||
acc_size = amdgpu_amdkfd_acc_size(size);
|
||||
|
||||
vram_needed = 0;
|
||||
if (domain == AMDGPU_GEM_DOMAIN_GTT) {
|
||||
/* TTM GTT memory */
|
||||
if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_GTT) {
|
||||
system_mem_needed = acc_size + size;
|
||||
ttm_mem_needed = acc_size + size;
|
||||
} else if (domain == AMDGPU_GEM_DOMAIN_CPU && !sg) {
|
||||
/* Userptr */
|
||||
system_mem_needed = acc_size + size;
|
||||
ttm_mem_needed = acc_size;
|
||||
} else {
|
||||
/* VRAM and SG */
|
||||
} else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) {
|
||||
system_mem_needed = acc_size;
|
||||
ttm_mem_needed = acc_size;
|
||||
if (domain == AMDGPU_GEM_DOMAIN_VRAM)
|
||||
vram_needed = size;
|
||||
vram_needed = size;
|
||||
} else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
|
||||
system_mem_needed = acc_size + size;
|
||||
ttm_mem_needed = acc_size;
|
||||
} else if (alloc_flag &
|
||||
(KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
|
||||
KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) {
|
||||
system_mem_needed = acc_size;
|
||||
ttm_mem_needed = acc_size;
|
||||
} else {
|
||||
pr_err("%s: Invalid BO type %#x\n", __func__, alloc_flag);
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
spin_lock(&kfd_mem_limit.mem_limit_lock);
|
||||
@ -166,64 +175,72 @@ static int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev,
|
||||
(adev->kfd.vram_used + vram_needed >
|
||||
adev->gmc.real_vram_size - reserved_for_pt)) {
|
||||
ret = -ENOMEM;
|
||||
} else {
|
||||
kfd_mem_limit.system_mem_used += system_mem_needed;
|
||||
kfd_mem_limit.ttm_mem_used += ttm_mem_needed;
|
||||
adev->kfd.vram_used += vram_needed;
|
||||
goto release;
|
||||
}
|
||||
|
||||
/* Update memory accounting by decreasing available system
|
||||
* memory, TTM memory and GPU memory as computed above
|
||||
*/
|
||||
adev->kfd.vram_used += vram_needed;
|
||||
kfd_mem_limit.system_mem_used += system_mem_needed;
|
||||
kfd_mem_limit.ttm_mem_used += ttm_mem_needed;
|
||||
|
||||
release:
|
||||
spin_unlock(&kfd_mem_limit.mem_limit_lock);
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void unreserve_mem_limit(struct amdgpu_device *adev,
|
||||
uint64_t size, u32 domain, bool sg)
|
||||
uint64_t size, u32 alloc_flag)
|
||||
{
|
||||
size_t acc_size;
|
||||
|
||||
acc_size = amdgpu_amdkfd_acc_size(size);
|
||||
|
||||
spin_lock(&kfd_mem_limit.mem_limit_lock);
|
||||
if (domain == AMDGPU_GEM_DOMAIN_GTT) {
|
||||
|
||||
if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_GTT) {
|
||||
kfd_mem_limit.system_mem_used -= (acc_size + size);
|
||||
kfd_mem_limit.ttm_mem_used -= (acc_size + size);
|
||||
} else if (domain == AMDGPU_GEM_DOMAIN_CPU && !sg) {
|
||||
kfd_mem_limit.system_mem_used -= (acc_size + size);
|
||||
kfd_mem_limit.ttm_mem_used -= acc_size;
|
||||
} else {
|
||||
} else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) {
|
||||
kfd_mem_limit.system_mem_used -= acc_size;
|
||||
kfd_mem_limit.ttm_mem_used -= acc_size;
|
||||
if (domain == AMDGPU_GEM_DOMAIN_VRAM) {
|
||||
adev->kfd.vram_used -= size;
|
||||
WARN_ONCE(adev->kfd.vram_used < 0,
|
||||
"kfd VRAM memory accounting unbalanced");
|
||||
}
|
||||
adev->kfd.vram_used -= size;
|
||||
} else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
|
||||
kfd_mem_limit.system_mem_used -= (acc_size + size);
|
||||
kfd_mem_limit.ttm_mem_used -= acc_size;
|
||||
} else if (alloc_flag &
|
||||
(KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
|
||||
KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) {
|
||||
kfd_mem_limit.system_mem_used -= acc_size;
|
||||
kfd_mem_limit.ttm_mem_used -= acc_size;
|
||||
} else {
|
||||
pr_err("%s: Invalid BO type %#x\n", __func__, alloc_flag);
|
||||
goto release;
|
||||
}
|
||||
WARN_ONCE(kfd_mem_limit.system_mem_used < 0,
|
||||
"kfd system memory accounting unbalanced");
|
||||
WARN_ONCE(kfd_mem_limit.ttm_mem_used < 0,
|
||||
"kfd TTM memory accounting unbalanced");
|
||||
|
||||
WARN_ONCE(adev->kfd.vram_used < 0,
|
||||
"KFD VRAM memory accounting unbalanced");
|
||||
WARN_ONCE(kfd_mem_limit.ttm_mem_used < 0,
|
||||
"KFD TTM memory accounting unbalanced");
|
||||
WARN_ONCE(kfd_mem_limit.system_mem_used < 0,
|
||||
"KFD system memory accounting unbalanced");
|
||||
|
||||
release:
|
||||
spin_unlock(&kfd_mem_limit.mem_limit_lock);
|
||||
}
|
||||
|
||||
void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo)
|
||||
{
|
||||
struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
|
||||
u32 domain = bo->preferred_domains;
|
||||
bool sg = (bo->preferred_domains == AMDGPU_GEM_DOMAIN_CPU);
|
||||
u32 alloc_flags = bo->kfd_bo->alloc_flags;
|
||||
u64 size = amdgpu_bo_size(bo);
|
||||
|
||||
if (bo->flags & AMDGPU_AMDKFD_CREATE_USERPTR_BO) {
|
||||
domain = AMDGPU_GEM_DOMAIN_CPU;
|
||||
sg = false;
|
||||
}
|
||||
|
||||
unreserve_mem_limit(adev, amdgpu_bo_size(bo), domain, sg);
|
||||
unreserve_mem_limit(adev, size, alloc_flags);
|
||||
|
||||
kfree(bo->kfd_bo);
|
||||
}
|
||||
|
||||
|
||||
/* amdgpu_amdkfd_remove_eviction_fence - Removes eviction fence from BO's
|
||||
* reservation object.
|
||||
*
|
||||
@ -646,12 +663,6 @@ kfd_mem_attach_dmabuf(struct amdgpu_device *adev, struct kgd_mem *mem,
|
||||
if (IS_ERR(gobj))
|
||||
return PTR_ERR(gobj);
|
||||
|
||||
/* Import takes an extra reference on the dmabuf. Drop it now to
|
||||
* avoid leaking it. We only need the one reference in
|
||||
* kgd_mem->dmabuf.
|
||||
*/
|
||||
dma_buf_put(mem->dmabuf);
|
||||
|
||||
*bo = gem_to_amdgpu_bo(gobj);
|
||||
(*bo)->flags |= AMDGPU_GEM_CREATE_PREEMPTIBLE;
|
||||
(*bo)->parent = amdgpu_bo_ref(mem->bo);
|
||||
@ -1278,12 +1289,60 @@ create_evict_fence_fail:
|
||||
return ret;
|
||||
}
|
||||
|
||||
int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct kgd_dev *kgd,
|
||||
/**
|
||||
* amdgpu_amdkfd_gpuvm_pin_bo() - Pins a BO using following criteria
|
||||
* @bo: Handle of buffer object being pinned
|
||||
* @domain: Domain into which BO should be pinned
|
||||
*
|
||||
* - USERPTR BOs are UNPINNABLE and will return error
|
||||
* - All other BO types (GTT, VRAM, MMIO and DOORBELL) will have their
|
||||
* PIN count incremented. It is valid to PIN a BO multiple times
|
||||
*
|
||||
* Return: ZERO if successful in pinning, Non-Zero in case of error.
|
||||
*/
|
||||
static int amdgpu_amdkfd_gpuvm_pin_bo(struct amdgpu_bo *bo, u32 domain)
|
||||
{
|
||||
int ret = 0;
|
||||
|
||||
ret = amdgpu_bo_reserve(bo, false);
|
||||
if (unlikely(ret))
|
||||
return ret;
|
||||
|
||||
ret = amdgpu_bo_pin_restricted(bo, domain, 0, 0);
|
||||
if (ret)
|
||||
pr_err("Error in Pinning BO to domain: %d\n", domain);
|
||||
|
||||
amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, false);
|
||||
amdgpu_bo_unreserve(bo);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* amdgpu_amdkfd_gpuvm_unpin_bo() - Unpins BO using following criteria
|
||||
* @bo: Handle of buffer object being unpinned
|
||||
*
|
||||
* - Is a illegal request for USERPTR BOs and is ignored
|
||||
* - All other BO types (GTT, VRAM, MMIO and DOORBELL) will have their
|
||||
* PIN count decremented. Calls to UNPIN must balance calls to PIN
|
||||
*/
|
||||
static void amdgpu_amdkfd_gpuvm_unpin_bo(struct amdgpu_bo *bo)
|
||||
{
|
||||
int ret = 0;
|
||||
|
||||
ret = amdgpu_bo_reserve(bo, false);
|
||||
if (unlikely(ret))
|
||||
return;
|
||||
|
||||
amdgpu_bo_unpin(bo);
|
||||
amdgpu_bo_unreserve(bo);
|
||||
}
|
||||
|
||||
int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct amdgpu_device *adev,
|
||||
struct file *filp, u32 pasid,
|
||||
void **process_info,
|
||||
struct dma_fence **ef)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
struct amdgpu_fpriv *drv_priv;
|
||||
struct amdgpu_vm *avm;
|
||||
int ret;
|
||||
@ -1359,12 +1418,12 @@ void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev,
|
||||
}
|
||||
}
|
||||
|
||||
void amdgpu_amdkfd_gpuvm_release_process_vm(struct kgd_dev *kgd, void *drm_priv)
|
||||
void amdgpu_amdkfd_gpuvm_release_process_vm(struct amdgpu_device *adev,
|
||||
void *drm_priv)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
struct amdgpu_vm *avm;
|
||||
|
||||
if (WARN_ON(!kgd || !drm_priv))
|
||||
if (WARN_ON(!adev || !drm_priv))
|
||||
return;
|
||||
|
||||
avm = drm_priv_to_vm(drm_priv);
|
||||
@ -1392,11 +1451,10 @@ uint64_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *drm_priv)
|
||||
}
|
||||
|
||||
int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
|
||||
struct kgd_dev *kgd, uint64_t va, uint64_t size,
|
||||
struct amdgpu_device *adev, uint64_t va, uint64_t size,
|
||||
void *drm_priv, struct kgd_mem **mem,
|
||||
uint64_t *offset, uint32_t flags)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);
|
||||
enum ttm_bo_type bo_type = ttm_bo_type_device;
|
||||
struct sg_table *sg = NULL;
|
||||
@ -1460,7 +1518,7 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
|
||||
|
||||
amdgpu_sync_create(&(*mem)->sync);
|
||||
|
||||
ret = amdgpu_amdkfd_reserve_mem_limit(adev, size, alloc_domain, !!sg);
|
||||
ret = amdgpu_amdkfd_reserve_mem_limit(adev, size, flags);
|
||||
if (ret) {
|
||||
pr_debug("Insufficient memory\n");
|
||||
goto err_reserve_limit;
|
||||
@ -1506,17 +1564,29 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
|
||||
if (offset)
|
||||
*offset = amdgpu_bo_mmap_offset(bo);
|
||||
|
||||
if (flags & (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
|
||||
KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) {
|
||||
ret = amdgpu_amdkfd_gpuvm_pin_bo(bo, AMDGPU_GEM_DOMAIN_GTT);
|
||||
if (ret) {
|
||||
pr_err("Pinning MMIO/DOORBELL BO during ALLOC FAILED\n");
|
||||
goto err_pin_bo;
|
||||
}
|
||||
bo->allowed_domains = AMDGPU_GEM_DOMAIN_GTT;
|
||||
bo->preferred_domains = AMDGPU_GEM_DOMAIN_GTT;
|
||||
}
|
||||
|
||||
return 0;
|
||||
|
||||
allocate_init_user_pages_failed:
|
||||
remove_kgd_mem_from_kfd_bo_list(*mem, avm->process_info);
|
||||
err_pin_bo:
|
||||
drm_vma_node_revoke(&gobj->vma_node, drm_priv);
|
||||
err_node_allow:
|
||||
drm_gem_object_put(gobj);
|
||||
/* Don't unreserve system mem limit twice */
|
||||
goto err_reserve_limit;
|
||||
err_bo_create:
|
||||
unreserve_mem_limit(adev, size, alloc_domain, !!sg);
|
||||
unreserve_mem_limit(adev, size, flags);
|
||||
err_reserve_limit:
|
||||
mutex_destroy(&(*mem)->lock);
|
||||
kfree(*mem);
|
||||
@ -1529,7 +1599,7 @@ err:
|
||||
}
|
||||
|
||||
int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
|
||||
struct kgd_dev *kgd, struct kgd_mem *mem, void *drm_priv,
|
||||
struct amdgpu_device *adev, struct kgd_mem *mem, void *drm_priv,
|
||||
uint64_t *size)
|
||||
{
|
||||
struct amdkfd_process_info *process_info = mem->process_info;
|
||||
@ -1542,6 +1612,14 @@ int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
|
||||
bool is_imported = false;
|
||||
|
||||
mutex_lock(&mem->lock);
|
||||
|
||||
/* Unpin MMIO/DOORBELL BO's that were pinnned during allocation */
|
||||
if (mem->alloc_flags &
|
||||
(KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
|
||||
KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) {
|
||||
amdgpu_amdkfd_gpuvm_unpin_bo(mem->bo);
|
||||
}
|
||||
|
||||
mapped_to_gpu_memory = mem->mapped_to_gpu_memory;
|
||||
is_imported = mem->is_imported;
|
||||
mutex_unlock(&mem->lock);
|
||||
@ -1621,10 +1699,9 @@ int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
|
||||
}
|
||||
|
||||
int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
|
||||
struct kgd_dev *kgd, struct kgd_mem *mem,
|
||||
struct amdgpu_device *adev, struct kgd_mem *mem,
|
||||
void *drm_priv, bool *table_freed)
|
||||
{
|
||||
struct amdgpu_device *adev = get_amdgpu_device(kgd);
|
||||
struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);
|
||||
int ret;
|
||||
struct amdgpu_bo *bo;
|
||||
@ -1751,7 +1828,7 @@ out:
|
||||
}
|
||||
|
||||
int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
|
||||
struct kgd_dev *kgd, struct kgd_mem *mem, void *drm_priv)
|
||||
struct amdgpu_device *adev, struct kgd_mem *mem, void *drm_priv)
|
||||
{
|
||||
struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);
|
||||
struct amdkfd_process_info *process_info = avm->process_info;
|
||||
@ -1812,7 +1889,7 @@ out:
|
||||
}
|
||||
|
||||
int amdgpu_amdkfd_gpuvm_sync_memory(
|
||||
struct kgd_dev *kgd, struct kgd_mem *mem, bool intr)
|
||||
struct amdgpu_device *adev, struct kgd_mem *mem, bool intr)
|
||||
{
|
||||
struct amdgpu_sync sync;
|
||||
int ret;
|
||||
@ -1828,7 +1905,7 @@ int amdgpu_amdkfd_gpuvm_sync_memory(
|
||||
return ret;
|
||||
}
|
||||
|
||||
int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_dev *kgd,
|
||||
int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct amdgpu_device *adev,
|
||||
struct kgd_mem *mem, void **kptr, uint64_t *size)
|
||||
{
|
||||
int ret;
|
||||
@ -1884,7 +1961,8 @@ bo_reserve_failed:
|
||||
return ret;
|
||||
}
|
||||
|
||||
void amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(struct kgd_dev *kgd, struct kgd_mem *mem)
|
||||
void amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(struct amdgpu_device *adev,
|
||||
struct kgd_mem *mem)
|
||||
{
|
||||
struct amdgpu_bo *bo = mem->bo;
|
||||
|
||||
@ -1894,12 +1972,9 @@ void amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(struct kgd_dev *kgd, struct kg
|
||||
amdgpu_bo_unreserve(bo);
|
||||
}
|
||||
|
||||
int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct kgd_dev *kgd,
|
||||
struct kfd_vm_fault_info *mem)
|
||||
int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct amdgpu_device *adev,
|
||||
struct kfd_vm_fault_info *mem)
|
||||
{
|
||||
struct amdgpu_device *adev;
|
||||
|
||||
adev = (struct amdgpu_device *)kgd;
|
||||
if (atomic_read(&adev->gmc.vm_fault_info_updated) == 1) {
|
||||
*mem = *adev->gmc.vm_fault_info;
|
||||
mb();
|
||||
@ -1908,13 +1983,12 @@ int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct kgd_dev *kgd,
|
||||
return 0;
|
||||
}
|
||||
|
||||
int amdgpu_amdkfd_gpuvm_import_dmabuf(struct kgd_dev *kgd,
|
||||
int amdgpu_amdkfd_gpuvm_import_dmabuf(struct amdgpu_device *adev,
|
||||
struct dma_buf *dma_buf,
|
||||
uint64_t va, void *drm_priv,
|
||||
struct kgd_mem **mem, uint64_t *size,
|
||||
uint64_t *mmap_offset)
|
||||
{
|
||||
struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
|
||||
struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);
|
||||
struct drm_gem_object *obj;
|
||||
struct amdgpu_bo *bo;
|
||||
@ -2541,11 +2615,9 @@ int amdgpu_amdkfd_remove_gws_from_process(void *info, void *mem)
|
||||
}
|
||||
|
||||
/* Returns GPU-specific tiling mode information */
|
||||
int amdgpu_amdkfd_get_tile_config(struct kgd_dev *kgd,
|
||||
int amdgpu_amdkfd_get_tile_config(struct amdgpu_device *adev,
|
||||
struct tile_config *config)
|
||||
{
|
||||
struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
|
||||
|
||||
config->gb_addr_config = adev->gfx.config.gb_addr_config;
|
||||
config->tile_config_ptr = adev->gfx.config.tile_mode_array;
|
||||
config->num_tile_configs =
|
||||
|
||||
@ -1569,6 +1569,18 @@ void amdgpu_atombios_scratch_regs_engine_hung(struct amdgpu_device *adev,
|
||||
WREG32(adev->bios_scratch_reg_offset + 3, tmp);
|
||||
}
|
||||
|
||||
void amdgpu_atombios_scratch_regs_set_backlight_level(struct amdgpu_device *adev,
|
||||
u32 backlight_level)
|
||||
{
|
||||
u32 tmp = RREG32(adev->bios_scratch_reg_offset + 2);
|
||||
|
||||
tmp &= ~ATOM_S2_CURRENT_BL_LEVEL_MASK;
|
||||
tmp |= (backlight_level << ATOM_S2_CURRENT_BL_LEVEL_SHIFT) &
|
||||
ATOM_S2_CURRENT_BL_LEVEL_MASK;
|
||||
|
||||
WREG32(adev->bios_scratch_reg_offset + 2, tmp);
|
||||
}
|
||||
|
||||
bool amdgpu_atombios_scratch_need_asic_init(struct amdgpu_device *adev)
|
||||
{
|
||||
u32 tmp = RREG32(adev->bios_scratch_reg_offset + 7);
|
||||
|
||||
@ -185,6 +185,8 @@ bool amdgpu_atombios_has_gpu_virtualization_table(struct amdgpu_device *adev);
|
||||
void amdgpu_atombios_scratch_regs_lock(struct amdgpu_device *adev, bool lock);
|
||||
void amdgpu_atombios_scratch_regs_engine_hung(struct amdgpu_device *adev,
|
||||
bool hung);
|
||||
void amdgpu_atombios_scratch_regs_set_backlight_level(struct amdgpu_device *adev,
|
||||
u32 backlight_level);
|
||||
bool amdgpu_atombios_scratch_need_asic_init(struct amdgpu_device *adev);
|
||||
|
||||
void amdgpu_atombios_copy_swap(u8 *dst, u8 *src, u8 num_bytes, bool to_le);
|
||||
|
||||
@ -3687,8 +3687,6 @@ fence_driver_init:
|
||||
/* Get a log2 for easy divisions. */
|
||||
adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));
|
||||
|
||||
amdgpu_fbdev_init(adev);
|
||||
|
||||
r = amdgpu_pm_sysfs_init(adev);
|
||||
if (r) {
|
||||
adev->pm_sysfs_en = false;
|
||||
@ -3846,8 +3844,6 @@ void amdgpu_device_fini_hw(struct amdgpu_device *adev)
|
||||
amdgpu_ucode_sysfs_fini(adev);
|
||||
sysfs_remove_files(&adev->dev->kobj, amdgpu_dev_attributes);
|
||||
|
||||
amdgpu_fbdev_fini(adev);
|
||||
|
||||
amdgpu_device_ip_fini_early(adev);
|
||||
|
||||
amdgpu_irq_fini_hw(adev);
|
||||
@ -3942,7 +3938,7 @@ int amdgpu_device_suspend(struct drm_device *dev, bool fbcon)
|
||||
drm_kms_helper_poll_disable(dev);
|
||||
|
||||
if (fbcon)
|
||||
amdgpu_fbdev_set_suspend(adev, 1);
|
||||
drm_fb_helper_set_suspend_unlocked(adev_to_drm(adev)->fb_helper, true);
|
||||
|
||||
cancel_delayed_work_sync(&adev->delayed_init_work);
|
||||
|
||||
@ -4019,7 +4015,7 @@ int amdgpu_device_resume(struct drm_device *dev, bool fbcon)
|
||||
flush_delayed_work(&adev->delayed_init_work);
|
||||
|
||||
if (fbcon)
|
||||
amdgpu_fbdev_set_suspend(adev, 0);
|
||||
drm_fb_helper_set_suspend_unlocked(adev_to_drm(adev)->fb_helper, false);
|
||||
|
||||
drm_kms_helper_poll_enable(dev);
|
||||
|
||||
@ -4316,7 +4312,6 @@ static int amdgpu_device_reset_sriov(struct amdgpu_device *adev,
|
||||
|
||||
amdgpu_irq_gpu_reset_resume_helper(adev);
|
||||
r = amdgpu_ib_ring_tests(adev);
|
||||
amdgpu_amdkfd_post_reset(adev);
|
||||
|
||||
error:
|
||||
if (!r && adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) {
|
||||
@ -4649,7 +4644,7 @@ int amdgpu_do_asic_reset(struct list_head *device_list_handle,
|
||||
if (r)
|
||||
goto out;
|
||||
|
||||
amdgpu_fbdev_set_suspend(tmp_adev, 0);
|
||||
drm_fb_helper_set_suspend_unlocked(adev_to_drm(tmp_adev)->fb_helper, false);
|
||||
|
||||
/*
|
||||
* The GPU enters bad state once faulty pages
|
||||
@ -5039,7 +5034,7 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
|
||||
*/
|
||||
amdgpu_unregister_gpu_instance(tmp_adev);
|
||||
|
||||
amdgpu_fbdev_set_suspend(tmp_adev, 1);
|
||||
drm_fb_helper_set_suspend_unlocked(adev_to_drm(adev)->fb_helper, true);
|
||||
|
||||
/* disable ras on ALL IPs */
|
||||
if (!need_emergency_restart &&
|
||||
@ -5089,7 +5084,7 @@ retry: /* Rest of adevs pre asic reset from XGMI hive. */
|
||||
|
||||
tmp_vram_lost_counter = atomic_read(&((adev)->vram_lost_counter));
|
||||
/* Actual ASIC resets if needed.*/
|
||||
/* TODO Implement XGMI hive reset logic for SRIOV */
|
||||
/* Host driver will handle XGMI hive reset for SRIOV */
|
||||
if (amdgpu_sriov_vf(adev)) {
|
||||
r = amdgpu_device_reset_sriov(adev, job ? false : true);
|
||||
if (r)
|
||||
@ -5149,8 +5144,8 @@ skip_hw_reset:
|
||||
|
||||
skip_sched_resume:
|
||||
list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
|
||||
/* unlock kfd: SRIOV would do it separately */
|
||||
if (!need_emergency_restart && !amdgpu_sriov_vf(tmp_adev))
|
||||
/* unlock kfd */
|
||||
if (!need_emergency_restart)
|
||||
amdgpu_amdkfd_post_reset(tmp_adev);
|
||||
|
||||
/* kfd_post_reset will do nothing if kfd device is not initialized,
|
||||
|
||||
@ -248,8 +248,8 @@ get_from_vram:
|
||||
|
||||
offset = offsetof(struct binary_header, binary_checksum) +
|
||||
sizeof(bhdr->binary_checksum);
|
||||
size = bhdr->binary_size - offset;
|
||||
checksum = bhdr->binary_checksum;
|
||||
size = le16_to_cpu(bhdr->binary_size) - offset;
|
||||
checksum = le16_to_cpu(bhdr->binary_checksum);
|
||||
|
||||
if (!amdgpu_discovery_verify_checksum(adev->mman.discovery_bin + offset,
|
||||
size, checksum)) {
|
||||
@ -270,7 +270,7 @@ get_from_vram:
|
||||
}
|
||||
|
||||
if (!amdgpu_discovery_verify_checksum(adev->mman.discovery_bin + offset,
|
||||
ihdr->size, checksum)) {
|
||||
le16_to_cpu(ihdr->size), checksum)) {
|
||||
DRM_ERROR("invalid ip discovery data table checksum\n");
|
||||
r = -EINVAL;
|
||||
goto out;
|
||||
@ -282,7 +282,7 @@ get_from_vram:
|
||||
ghdr = (struct gpu_info_header *)(adev->mman.discovery_bin + offset);
|
||||
|
||||
if (!amdgpu_discovery_verify_checksum(adev->mman.discovery_bin + offset,
|
||||
ghdr->size, checksum)) {
|
||||
le32_to_cpu(ghdr->size), checksum)) {
|
||||
DRM_ERROR("invalid gc data table checksum\n");
|
||||
r = -EINVAL;
|
||||
goto out;
|
||||
@ -489,10 +489,10 @@ void amdgpu_discovery_harvest_ip(struct amdgpu_device *adev)
|
||||
le16_to_cpu(bhdr->table_list[HARVEST_INFO].offset));
|
||||
|
||||
for (i = 0; i < 32; i++) {
|
||||
if (le32_to_cpu(harvest_info->list[i].hw_id) == 0)
|
||||
if (le16_to_cpu(harvest_info->list[i].hw_id) == 0)
|
||||
break;
|
||||
|
||||
switch (le32_to_cpu(harvest_info->list[i].hw_id)) {
|
||||
switch (le16_to_cpu(harvest_info->list[i].hw_id)) {
|
||||
case VCN_HWID:
|
||||
vcn_harvest_count++;
|
||||
if (harvest_info->list[i].number_instance == 0)
|
||||
|
||||
@ -1599,13 +1599,10 @@ int amdgpu_display_suspend_helper(struct amdgpu_device *adev)
|
||||
continue;
|
||||
}
|
||||
robj = gem_to_amdgpu_bo(fb->obj[0]);
|
||||
/* don't unpin kernel fb objects */
|
||||
if (!amdgpu_fbdev_robj_is_fb(adev, robj)) {
|
||||
r = amdgpu_bo_reserve(robj, true);
|
||||
if (r == 0) {
|
||||
amdgpu_bo_unpin(robj);
|
||||
amdgpu_bo_unreserve(robj);
|
||||
}
|
||||
r = amdgpu_bo_reserve(robj, true);
|
||||
if (r == 0) {
|
||||
amdgpu_bo_unpin(robj);
|
||||
amdgpu_bo_unreserve(robj);
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
|
||||
@ -2001,6 +2001,19 @@ retry_init:
|
||||
goto err_pci;
|
||||
}
|
||||
|
||||
/*
|
||||
* 1. don't init fbdev on hw without DCE
|
||||
* 2. don't init fbdev if there are no connectors
|
||||
*/
|
||||
if (adev->mode_info.mode_config_initialized &&
|
||||
!list_empty(&adev_to_drm(adev)->mode_config.connector_list)) {
|
||||
/* select 8 bpp console on low vram cards */
|
||||
if (adev->gmc.real_vram_size <= (32*1024*1024))
|
||||
drm_fbdev_generic_setup(adev_to_drm(adev), 8);
|
||||
else
|
||||
drm_fbdev_generic_setup(adev_to_drm(adev), 32);
|
||||
}
|
||||
|
||||
ret = amdgpu_debugfs_init(adev);
|
||||
if (ret)
|
||||
DRM_ERROR("Creating debugfs files failed (%d).\n", ret);
|
||||
|
||||
@ -1,388 +0,0 @@
|
||||
/*
|
||||
* Copyright © 2007 David Airlie
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including the next
|
||||
* paragraph) shall be included in all copies or substantial portions of the
|
||||
* Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
* DEALINGS IN THE SOFTWARE.
|
||||
*
|
||||
* Authors:
|
||||
* David Airlie
|
||||
*/
|
||||
|
||||
#include <linux/module.h>
|
||||
#include <linux/pm_runtime.h>
|
||||
#include <linux/slab.h>
|
||||
#include <linux/vga_switcheroo.h>
|
||||
|
||||
#include <drm/amdgpu_drm.h>
|
||||
#include <drm/drm_crtc.h>
|
||||
#include <drm/drm_crtc_helper.h>
|
||||
#include <drm/drm_fb_helper.h>
|
||||
#include <drm/drm_fourcc.h>
|
||||
|
||||
#include "amdgpu.h"
|
||||
#include "cikd.h"
|
||||
#include "amdgpu_gem.h"
|
||||
|
||||
#include "amdgpu_display.h"
|
||||
|
||||
/* object hierarchy -
|
||||
this contains a helper + a amdgpu fb
|
||||
the helper contains a pointer to amdgpu framebuffer baseclass.
|
||||
*/
|
||||
|
||||
static int
|
||||
amdgpufb_open(struct fb_info *info, int user)
|
||||
{
|
||||
struct drm_fb_helper *fb_helper = info->par;
|
||||
int ret = pm_runtime_get_sync(fb_helper->dev->dev);
|
||||
if (ret < 0 && ret != -EACCES) {
|
||||
pm_runtime_mark_last_busy(fb_helper->dev->dev);
|
||||
pm_runtime_put_autosuspend(fb_helper->dev->dev);
|
||||
return ret;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int
|
||||
amdgpufb_release(struct fb_info *info, int user)
|
||||
{
|
||||
struct drm_fb_helper *fb_helper = info->par;
|
||||
|
||||
pm_runtime_mark_last_busy(fb_helper->dev->dev);
|
||||
pm_runtime_put_autosuspend(fb_helper->dev->dev);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static const struct fb_ops amdgpufb_ops = {
|
||||
.owner = THIS_MODULE,
|
||||
DRM_FB_HELPER_DEFAULT_OPS,
|
||||
.fb_open = amdgpufb_open,
|
||||
.fb_release = amdgpufb_release,
|
||||
.fb_fillrect = drm_fb_helper_cfb_fillrect,
|
||||
.fb_copyarea = drm_fb_helper_cfb_copyarea,
|
||||
.fb_imageblit = drm_fb_helper_cfb_imageblit,
|
||||
};
|
||||
|
||||
|
||||
int amdgpu_align_pitch(struct amdgpu_device *adev, int width, int cpp, bool tiled)
|
||||
{
|
||||
int aligned = width;
|
||||
int pitch_mask = 0;
|
||||
|
||||
switch (cpp) {
|
||||
case 1:
|
||||
pitch_mask = 255;
|
||||
break;
|
||||
case 2:
|
||||
pitch_mask = 127;
|
||||
break;
|
||||
case 3:
|
||||
case 4:
|
||||
pitch_mask = 63;
|
||||
break;
|
||||
}
|
||||
|
||||
aligned += pitch_mask;
|
||||
aligned &= ~pitch_mask;
|
||||
return aligned * cpp;
|
||||
}
|
||||
|
||||
static void amdgpufb_destroy_pinned_object(struct drm_gem_object *gobj)
|
||||
{
|
||||
struct amdgpu_bo *abo = gem_to_amdgpu_bo(gobj);
|
||||
int ret;
|
||||
|
||||
ret = amdgpu_bo_reserve(abo, true);
|
||||
if (likely(ret == 0)) {
|
||||
amdgpu_bo_kunmap(abo);
|
||||
amdgpu_bo_unpin(abo);
|
||||
amdgpu_bo_unreserve(abo);
|
||||
}
|
||||
drm_gem_object_put(gobj);
|
||||
}
|
||||
|
||||
static int amdgpufb_create_pinned_object(struct amdgpu_fbdev *rfbdev,
|
||||
struct drm_mode_fb_cmd2 *mode_cmd,
|
||||
struct drm_gem_object **gobj_p)
|
||||
{
|
||||
const struct drm_format_info *info;
|
||||
struct amdgpu_device *adev = rfbdev->adev;
|
||||
struct drm_gem_object *gobj = NULL;
|
||||
struct amdgpu_bo *abo = NULL;
|
||||
bool fb_tiled = false; /* useful for testing */
|
||||
u32 tiling_flags = 0, domain;
|
||||
int ret;
|
||||
int aligned_size, size;
|
||||
int height = mode_cmd->height;
|
||||
u32 cpp;
|
||||
u64 flags = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
|
||||
AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS |
|
||||
AMDGPU_GEM_CREATE_VRAM_CLEARED;
|
||||
|
||||
info = drm_get_format_info(adev_to_drm(adev), mode_cmd);
|
||||
cpp = info->cpp[0];
|
||||
|
||||
/* need to align pitch with crtc limits */
|
||||
mode_cmd->pitches[0] = amdgpu_align_pitch(adev, mode_cmd->width, cpp,
|
||||
fb_tiled);
|
||||
domain = amdgpu_display_supported_domains(adev, flags);
|
||||
height = ALIGN(mode_cmd->height, 8);
|
||||
size = mode_cmd->pitches[0] * height;
|
||||
aligned_size = ALIGN(size, PAGE_SIZE);
|
||||
ret = amdgpu_gem_object_create(adev, aligned_size, 0, domain, flags,
|
||||
ttm_bo_type_device, NULL, &gobj);
|
||||
if (ret) {
|
||||
pr_err("failed to allocate framebuffer (%d)\n", aligned_size);
|
||||
return -ENOMEM;
|
||||
}
|
||||
abo = gem_to_amdgpu_bo(gobj);
|
||||
|
||||
if (fb_tiled)
|
||||
tiling_flags = AMDGPU_TILING_SET(ARRAY_MODE, GRPH_ARRAY_2D_TILED_THIN1);
|
||||
|
||||
ret = amdgpu_bo_reserve(abo, false);
|
||||
if (unlikely(ret != 0))
|
||||
goto out_unref;
|
||||
|
||||
if (tiling_flags) {
|
||||
ret = amdgpu_bo_set_tiling_flags(abo,
|
||||
tiling_flags);
|
||||
if (ret)
|
||||
dev_err(adev->dev, "FB failed to set tiling flags\n");
|
||||
}
|
||||
|
||||
ret = amdgpu_bo_pin(abo, domain);
|
||||
if (ret) {
|
||||
amdgpu_bo_unreserve(abo);
|
||||
goto out_unref;
|
||||
}
|
||||
|
||||
ret = amdgpu_ttm_alloc_gart(&abo->tbo);
|
||||
if (ret) {
|
||||
amdgpu_bo_unreserve(abo);
|
||||
dev_err(adev->dev, "%p bind failed\n", abo);
|
||||
goto out_unref;
|
||||
}
|
||||
|
||||
ret = amdgpu_bo_kmap(abo, NULL);
|
||||
amdgpu_bo_unreserve(abo);
|
||||
if (ret) {
|
||||
goto out_unref;
|
||||
}
|
||||
|
||||
*gobj_p = gobj;
|
||||
return 0;
|
||||
out_unref:
|
||||
amdgpufb_destroy_pinned_object(gobj);
|
||||
*gobj_p = NULL;
|
||||
return ret;
|
||||
}
|
||||
|
||||
static int amdgpufb_create(struct drm_fb_helper *helper,
|
||||
struct drm_fb_helper_surface_size *sizes)
|
||||
{
|
||||
struct amdgpu_fbdev *rfbdev = (struct amdgpu_fbdev *)helper;
|
||||
struct amdgpu_device *adev = rfbdev->adev;
|
||||
struct fb_info *info;
|
||||
struct drm_framebuffer *fb = NULL;
|
||||
struct drm_mode_fb_cmd2 mode_cmd;
|
||||
struct drm_gem_object *gobj = NULL;
|
||||
struct amdgpu_bo *abo = NULL;
|
||||
int ret;
|
||||
|
||||
memset(&mode_cmd, 0, sizeof(mode_cmd));
|
||||
mode_cmd.width = sizes->surface_width;
|
||||
mode_cmd.height = sizes->surface_height;
|
||||
|
||||
if (sizes->surface_bpp == 24)
|
||||
sizes->surface_bpp = 32;
|
||||
|
||||
mode_cmd.pixel_format = drm_mode_legacy_fb_format(sizes->surface_bpp,
|
||||
sizes->surface_depth);
|
||||
|
||||
ret = amdgpufb_create_pinned_object(rfbdev, &mode_cmd, &gobj);
|
||||
if (ret) {
|
||||
DRM_ERROR("failed to create fbcon object %d\n", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
abo = gem_to_amdgpu_bo(gobj);
|
||||
|
||||
/* okay we have an object now allocate the framebuffer */
|
||||
info = drm_fb_helper_alloc_fbi(helper);
|
||||
if (IS_ERR(info)) {
|
||||
ret = PTR_ERR(info);
|
||||
goto out;
|
||||
}
|
||||
|
||||
ret = amdgpu_display_gem_fb_init(adev_to_drm(adev), &rfbdev->rfb,
|
||||
&mode_cmd, gobj);
|
||||
if (ret) {
|
||||
DRM_ERROR("failed to initialize framebuffer %d\n", ret);
|
||||
goto out;
|
||||
}
|
||||
|
||||
fb = &rfbdev->rfb.base;
|
||||
|
||||
/* setup helper */
|
||||
rfbdev->helper.fb = fb;
|
||||
|
||||
info->fbops = &amdgpufb_ops;
|
||||
|
||||
info->fix.smem_start = amdgpu_gmc_vram_cpu_pa(adev, abo);
|
||||
info->fix.smem_len = amdgpu_bo_size(abo);
|
||||
info->screen_base = amdgpu_bo_kptr(abo);
|
||||
info->screen_size = amdgpu_bo_size(abo);
|
||||
|
||||
drm_fb_helper_fill_info(info, &rfbdev->helper, sizes);
|
||||
|
||||
/* setup aperture base/size for vesafb takeover */
|
||||
info->apertures->ranges[0].base = adev_to_drm(adev)->mode_config.fb_base;
|
||||
info->apertures->ranges[0].size = adev->gmc.aper_size;
|
||||
|
||||
/* Use default scratch pixmap (info->pixmap.flags = FB_PIXMAP_SYSTEM) */
|
||||
|
||||
if (info->screen_base == NULL) {
|
||||
ret = -ENOSPC;
|
||||
goto out;
|
||||
}
|
||||
|
||||
DRM_INFO("fb mappable at 0x%lX\n", info->fix.smem_start);
|
||||
DRM_INFO("vram apper at 0x%lX\n", (unsigned long)adev->gmc.aper_base);
|
||||
DRM_INFO("size %lu\n", (unsigned long)amdgpu_bo_size(abo));
|
||||
DRM_INFO("fb depth is %d\n", fb->format->depth);
|
||||
DRM_INFO(" pitch is %d\n", fb->pitches[0]);
|
||||
|
||||
vga_switcheroo_client_fb_set(adev->pdev, info);
|
||||
return 0;
|
||||
|
||||
out:
|
||||
if (fb && ret) {
|
||||
drm_gem_object_put(gobj);
|
||||
drm_framebuffer_unregister_private(fb);
|
||||
drm_framebuffer_cleanup(fb);
|
||||
kfree(fb);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
static int amdgpu_fbdev_destroy(struct drm_device *dev, struct amdgpu_fbdev *rfbdev)
|
||||
{
|
||||
struct amdgpu_framebuffer *rfb = &rfbdev->rfb;
|
||||
int i;
|
||||
|
||||
drm_fb_helper_unregister_fbi(&rfbdev->helper);
|
||||
|
||||
if (rfb->base.obj[0]) {
|
||||
for (i = 0; i < rfb->base.format->num_planes; i++)
|
||||
drm_gem_object_put(rfb->base.obj[0]);
|
||||
amdgpufb_destroy_pinned_object(rfb->base.obj[0]);
|
||||
rfb->base.obj[0] = NULL;
|
||||
drm_framebuffer_unregister_private(&rfb->base);
|
||||
drm_framebuffer_cleanup(&rfb->base);
|
||||
}
|
||||
drm_fb_helper_fini(&rfbdev->helper);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static const struct drm_fb_helper_funcs amdgpu_fb_helper_funcs = {
|
||||
.fb_probe = amdgpufb_create,
|
||||
};
|
||||
|
||||
int amdgpu_fbdev_init(struct amdgpu_device *adev)
|
||||
{
|
||||
struct amdgpu_fbdev *rfbdev;
|
||||
int bpp_sel = 32;
|
||||
int ret;
|
||||
|
||||
/* don't init fbdev on hw without DCE */
|
||||
if (!adev->mode_info.mode_config_initialized)
|
||||
return 0;
|
||||
|
||||
/* don't init fbdev if there are no connectors */
|
||||
if (list_empty(&adev_to_drm(adev)->mode_config.connector_list))
|
||||
return 0;
|
||||
|
||||
/* select 8 bpp console on low vram cards */
|
||||
if (adev->gmc.real_vram_size <= (32*1024*1024))
|
||||
bpp_sel = 8;
|
||||
|
||||
rfbdev = kzalloc(sizeof(struct amdgpu_fbdev), GFP_KERNEL);
|
||||
if (!rfbdev)
|
||||
return -ENOMEM;
|
||||
|
||||
rfbdev->adev = adev;
|
||||
adev->mode_info.rfbdev = rfbdev;
|
||||
|
||||
drm_fb_helper_prepare(adev_to_drm(adev), &rfbdev->helper,
|
||||
&amdgpu_fb_helper_funcs);
|
||||
|
||||
ret = drm_fb_helper_init(adev_to_drm(adev), &rfbdev->helper);
|
||||
if (ret) {
|
||||
kfree(rfbdev);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/* disable all the possible outputs/crtcs before entering KMS mode */
|
||||
if (!amdgpu_device_has_dc_support(adev) && !amdgpu_virtual_display)
|
||||
drm_helper_disable_unused_functions(adev_to_drm(adev));
|
||||
|
||||
drm_fb_helper_initial_config(&rfbdev->helper, bpp_sel);
|
||||
return 0;
|
||||
}
|
||||
|
||||
void amdgpu_fbdev_fini(struct amdgpu_device *adev)
|
||||
{
|
||||
if (!adev->mode_info.rfbdev)
|
||||
return;
|
||||
|
||||
amdgpu_fbdev_destroy(adev_to_drm(adev), adev->mode_info.rfbdev);
|
||||
kfree(adev->mode_info.rfbdev);
|
||||
adev->mode_info.rfbdev = NULL;
|
||||
}
|
||||
|
||||
void amdgpu_fbdev_set_suspend(struct amdgpu_device *adev, int state)
|
||||
{
|
||||
if (adev->mode_info.rfbdev)
|
||||
drm_fb_helper_set_suspend_unlocked(&adev->mode_info.rfbdev->helper,
|
||||
state);
|
||||
}
|
||||
|
||||
int amdgpu_fbdev_total_size(struct amdgpu_device *adev)
|
||||
{
|
||||
struct amdgpu_bo *robj;
|
||||
int size = 0;
|
||||
|
||||
if (!adev->mode_info.rfbdev)
|
||||
return 0;
|
||||
|
||||
robj = gem_to_amdgpu_bo(adev->mode_info.rfbdev->rfb.base.obj[0]);
|
||||
size += amdgpu_bo_size(robj);
|
||||
return size;
|
||||
}
|
||||
|
||||
bool amdgpu_fbdev_robj_is_fb(struct amdgpu_device *adev, struct amdgpu_bo *robj)
|
||||
{
|
||||
if (!adev->mode_info.rfbdev)
|
||||
return false;
|
||||
if (robj == gem_to_amdgpu_bo(adev->mode_info.rfbdev->rfb.base.obj[0]))
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
@ -877,6 +877,32 @@ out:
|
||||
return r;
|
||||
}
|
||||
|
||||
static int amdgpu_gem_align_pitch(struct amdgpu_device *adev,
|
||||
int width,
|
||||
int cpp,
|
||||
bool tiled)
|
||||
{
|
||||
int aligned = width;
|
||||
int pitch_mask = 0;
|
||||
|
||||
switch (cpp) {
|
||||
case 1:
|
||||
pitch_mask = 255;
|
||||
break;
|
||||
case 2:
|
||||
pitch_mask = 127;
|
||||
break;
|
||||
case 3:
|
||||
case 4:
|
||||
pitch_mask = 63;
|
||||
break;
|
||||
}
|
||||
|
||||
aligned += pitch_mask;
|
||||
aligned &= ~pitch_mask;
|
||||
return aligned * cpp;
|
||||
}
|
||||
|
||||
int amdgpu_mode_dumb_create(struct drm_file *file_priv,
|
||||
struct drm_device *dev,
|
||||
struct drm_mode_create_dumb *args)
|
||||
@ -885,7 +911,8 @@ int amdgpu_mode_dumb_create(struct drm_file *file_priv,
|
||||
struct drm_gem_object *gobj;
|
||||
uint32_t handle;
|
||||
u64 flags = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
|
||||
AMDGPU_GEM_CREATE_CPU_GTT_USWC;
|
||||
AMDGPU_GEM_CREATE_CPU_GTT_USWC |
|
||||
AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
|
||||
u32 domain;
|
||||
int r;
|
||||
|
||||
@ -897,8 +924,8 @@ int amdgpu_mode_dumb_create(struct drm_file *file_priv,
|
||||
if (adev->mman.buffer_funcs_enabled)
|
||||
flags |= AMDGPU_GEM_CREATE_VRAM_CLEARED;
|
||||
|
||||
args->pitch = amdgpu_align_pitch(adev, args->width,
|
||||
DIV_ROUND_UP(args->bpp, 8), 0);
|
||||
args->pitch = amdgpu_gem_align_pitch(adev, args->width,
|
||||
DIV_ROUND_UP(args->bpp, 8), 0);
|
||||
args->size = (u64)args->pitch * args->height;
|
||||
args->size = ALIGN(args->size, PAGE_SIZE);
|
||||
domain = amdgpu_bo_get_preferred_domain(adev,
|
||||
|
||||
@ -223,7 +223,7 @@ int amdgpu_ih_wait_on_checkpoint_process(struct amdgpu_device *adev,
|
||||
*/
|
||||
int amdgpu_ih_process(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih)
|
||||
{
|
||||
unsigned int count = AMDGPU_IH_MAX_NUM_IVS;
|
||||
unsigned int count;
|
||||
u32 wptr;
|
||||
|
||||
if (!ih->enabled || adev->shutdown)
|
||||
@ -232,6 +232,7 @@ int amdgpu_ih_process(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih)
|
||||
wptr = amdgpu_ih_get_wptr(adev, ih);
|
||||
|
||||
restart_ih:
|
||||
count = AMDGPU_IH_MAX_NUM_IVS;
|
||||
DRM_DEBUG("%s: rptr %d, wptr %d\n", __func__, ih->rptr, wptr);
|
||||
|
||||
/* Order reading of wptr vs. reading of IH ring data */
|
||||
|
||||
@ -333,7 +333,6 @@ int amdgpu_irq_init(struct amdgpu_device *adev)
|
||||
if (!amdgpu_device_has_dc_support(adev)) {
|
||||
if (!adev->enable_virtual_display)
|
||||
/* Disable vblank IRQs aggressively for power-saving */
|
||||
/* XXX: can this be enabled for DC? */
|
||||
adev_to_drm(adev)->vblank_disable_immediate = true;
|
||||
|
||||
r = drm_vblank_init(adev_to_drm(adev), adev->mode_info.num_crtc);
|
||||
|
||||
@ -232,8 +232,6 @@ struct amdgpu_i2c_chan {
|
||||
struct mutex mutex;
|
||||
};
|
||||
|
||||
struct amdgpu_fbdev;
|
||||
|
||||
struct amdgpu_afmt {
|
||||
bool enabled;
|
||||
int offset;
|
||||
@ -309,13 +307,6 @@ struct amdgpu_framebuffer {
|
||||
uint64_t address;
|
||||
};
|
||||
|
||||
struct amdgpu_fbdev {
|
||||
struct drm_fb_helper helper;
|
||||
struct amdgpu_framebuffer rfb;
|
||||
struct list_head fbdev_list;
|
||||
struct amdgpu_device *adev;
|
||||
};
|
||||
|
||||
struct amdgpu_mode_info {
|
||||
struct atom_context *atom_context;
|
||||
struct card_info *atom_card_info;
|
||||
@ -341,8 +332,6 @@ struct amdgpu_mode_info {
|
||||
struct edid *bios_hardcoded_edid;
|
||||
int bios_hardcoded_edid_size;
|
||||
|
||||
/* pointer to fbdev info structure */
|
||||
struct amdgpu_fbdev *rfbdev;
|
||||
/* firmware flags */
|
||||
u32 firmware_flags;
|
||||
/* pointer to backlight encoder */
|
||||
@ -631,15 +620,6 @@ bool amdgpu_crtc_get_scanout_position(struct drm_crtc *crtc,
|
||||
int *hpos, ktime_t *stime, ktime_t *etime,
|
||||
const struct drm_display_mode *mode);
|
||||
|
||||
/* fbdev layer */
|
||||
int amdgpu_fbdev_init(struct amdgpu_device *adev);
|
||||
void amdgpu_fbdev_fini(struct amdgpu_device *adev);
|
||||
void amdgpu_fbdev_set_suspend(struct amdgpu_device *adev, int state);
|
||||
int amdgpu_fbdev_total_size(struct amdgpu_device *adev);
|
||||
bool amdgpu_fbdev_robj_is_fb(struct amdgpu_device *adev, struct amdgpu_bo *robj);
|
||||
|
||||
int amdgpu_align_pitch(struct amdgpu_device *adev, int width, int bpp, bool tiled);
|
||||
|
||||
/* amdgpu_display.c */
|
||||
void amdgpu_display_print_display_setup(struct drm_device *dev);
|
||||
int amdgpu_display_modeset_create_props(struct amdgpu_device *adev);
|
||||
|
||||
@ -1032,9 +1032,14 @@ int amdgpu_bo_init(struct amdgpu_device *adev)
|
||||
/* On A+A platform, VRAM can be mapped as WB */
|
||||
if (!adev->gmc.xgmi.connected_to_cpu) {
|
||||
/* reserve PAT memory space to WC for VRAM */
|
||||
arch_io_reserve_memtype_wc(adev->gmc.aper_base,
|
||||
int r = arch_io_reserve_memtype_wc(adev->gmc.aper_base,
|
||||
adev->gmc.aper_size);
|
||||
|
||||
if (r) {
|
||||
DRM_ERROR("Unable to set WC memtype for the aperture base\n");
|
||||
return r;
|
||||
}
|
||||
|
||||
/* Add an MTRR for the VRAM */
|
||||
adev->gmc.vram_mtrr = arch_phys_wc_add(adev->gmc.aper_base,
|
||||
adev->gmc.aper_size);
|
||||
|
||||
@ -892,6 +892,38 @@ void amdgpu_ras_mca_query_error_status(struct amdgpu_device *adev,
|
||||
}
|
||||
}
|
||||
|
||||
static void amdgpu_ras_get_ecc_info(struct amdgpu_device *adev, struct ras_err_data *err_data)
|
||||
{
|
||||
struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
|
||||
int ret = 0;
|
||||
|
||||
/*
|
||||
* choosing right query method according to
|
||||
* whether smu support query error information
|
||||
*/
|
||||
ret = smu_get_ecc_info(&adev->smu, (void *)&(ras->umc_ecc));
|
||||
if (ret == -EOPNOTSUPP) {
|
||||
if (adev->umc.ras_funcs &&
|
||||
adev->umc.ras_funcs->query_ras_error_count)
|
||||
adev->umc.ras_funcs->query_ras_error_count(adev, err_data);
|
||||
|
||||
/* umc query_ras_error_address is also responsible for clearing
|
||||
* error status
|
||||
*/
|
||||
if (adev->umc.ras_funcs &&
|
||||
adev->umc.ras_funcs->query_ras_error_address)
|
||||
adev->umc.ras_funcs->query_ras_error_address(adev, err_data);
|
||||
} else if (!ret) {
|
||||
if (adev->umc.ras_funcs &&
|
||||
adev->umc.ras_funcs->ecc_info_query_ras_error_count)
|
||||
adev->umc.ras_funcs->ecc_info_query_ras_error_count(adev, err_data);
|
||||
|
||||
if (adev->umc.ras_funcs &&
|
||||
adev->umc.ras_funcs->ecc_info_query_ras_error_address)
|
||||
adev->umc.ras_funcs->ecc_info_query_ras_error_address(adev, err_data);
|
||||
}
|
||||
}
|
||||
|
||||
/* query/inject/cure begin */
|
||||
int amdgpu_ras_query_error_status(struct amdgpu_device *adev,
|
||||
struct ras_query_if *info)
|
||||
@ -905,15 +937,7 @@ int amdgpu_ras_query_error_status(struct amdgpu_device *adev,
|
||||
|
||||
switch (info->head.block) {
|
||||
case AMDGPU_RAS_BLOCK__UMC:
|
||||
if (adev->umc.ras_funcs &&
|
||||
adev->umc.ras_funcs->query_ras_error_count)
|
||||
adev->umc.ras_funcs->query_ras_error_count(adev, &err_data);
|
||||
/* umc query_ras_error_address is also responsible for clearing
|
||||
* error status
|
||||
*/
|
||||
if (adev->umc.ras_funcs &&
|
||||
adev->umc.ras_funcs->query_ras_error_address)
|
||||
adev->umc.ras_funcs->query_ras_error_address(adev, &err_data);
|
||||
amdgpu_ras_get_ecc_info(adev, &err_data);
|
||||
break;
|
||||
case AMDGPU_RAS_BLOCK__SDMA:
|
||||
if (adev->sdma.funcs->query_ras_error_count) {
|
||||
@ -1935,9 +1959,11 @@ int amdgpu_ras_save_bad_pages(struct amdgpu_device *adev)
|
||||
if (!con || !con->eh_data)
|
||||
return 0;
|
||||
|
||||
mutex_lock(&con->recovery_lock);
|
||||
control = &con->eeprom_control;
|
||||
data = con->eh_data;
|
||||
save_count = data->count - control->ras_num_recs;
|
||||
mutex_unlock(&con->recovery_lock);
|
||||
/* only new entries are saved */
|
||||
if (save_count > 0) {
|
||||
if (amdgpu_ras_eeprom_append(control,
|
||||
|
||||
@ -319,6 +319,19 @@ struct ras_common_if {
|
||||
char name[32];
|
||||
};
|
||||
|
||||
#define MAX_UMC_CHANNEL_NUM 32
|
||||
|
||||
struct ecc_info_per_ch {
|
||||
uint16_t ce_count_lo_chip;
|
||||
uint16_t ce_count_hi_chip;
|
||||
uint64_t mca_umc_status;
|
||||
uint64_t mca_umc_addr;
|
||||
};
|
||||
|
||||
struct umc_ecc_info {
|
||||
struct ecc_info_per_ch ecc[MAX_UMC_CHANNEL_NUM];
|
||||
};
|
||||
|
||||
struct amdgpu_ras {
|
||||
/* ras infrastructure */
|
||||
/* for ras itself. */
|
||||
@ -358,6 +371,9 @@ struct amdgpu_ras {
|
||||
struct delayed_work ras_counte_delay_work;
|
||||
atomic_t ras_ue_count;
|
||||
atomic_t ras_ce_count;
|
||||
|
||||
/* record umc error info queried from smu */
|
||||
struct umc_ecc_info umc_ecc;
|
||||
};
|
||||
|
||||
struct ras_fs_data {
|
||||
|
||||
@ -913,11 +913,6 @@ static int amdgpu_ttm_backend_bind(struct ttm_device *bdev,
|
||||
ttm->num_pages, bo_mem, ttm);
|
||||
}
|
||||
|
||||
if (bo_mem->mem_type == AMDGPU_PL_GDS ||
|
||||
bo_mem->mem_type == AMDGPU_PL_GWS ||
|
||||
bo_mem->mem_type == AMDGPU_PL_OA)
|
||||
return -EINVAL;
|
||||
|
||||
if (bo_mem->mem_type != TTM_PL_TT ||
|
||||
!amdgpu_gtt_mgr_has_gart_addr(bo_mem)) {
|
||||
gtt->offset = AMDGPU_BO_INVALID_OFFSET;
|
||||
|
||||
@ -94,30 +94,58 @@ int amdgpu_umc_process_ras_data_cb(struct amdgpu_device *adev,
|
||||
{
|
||||
struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
|
||||
struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
|
||||
int ret = 0;
|
||||
|
||||
kgd2kfd_set_sram_ecc_flag(adev->kfd.dev);
|
||||
if (adev->umc.ras_funcs &&
|
||||
adev->umc.ras_funcs->query_ras_error_count)
|
||||
adev->umc.ras_funcs->query_ras_error_count(adev, ras_error_status);
|
||||
ret = smu_get_ecc_info(&adev->smu, (void *)&(con->umc_ecc));
|
||||
if (ret == -EOPNOTSUPP) {
|
||||
if (adev->umc.ras_funcs &&
|
||||
adev->umc.ras_funcs->query_ras_error_count)
|
||||
adev->umc.ras_funcs->query_ras_error_count(adev, ras_error_status);
|
||||
|
||||
if (adev->umc.ras_funcs &&
|
||||
adev->umc.ras_funcs->query_ras_error_address &&
|
||||
adev->umc.max_ras_err_cnt_per_query) {
|
||||
err_data->err_addr =
|
||||
kcalloc(adev->umc.max_ras_err_cnt_per_query,
|
||||
sizeof(struct eeprom_table_record), GFP_KERNEL);
|
||||
if (adev->umc.ras_funcs &&
|
||||
adev->umc.ras_funcs->query_ras_error_address &&
|
||||
adev->umc.max_ras_err_cnt_per_query) {
|
||||
err_data->err_addr =
|
||||
kcalloc(adev->umc.max_ras_err_cnt_per_query,
|
||||
sizeof(struct eeprom_table_record), GFP_KERNEL);
|
||||
|
||||
/* still call query_ras_error_address to clear error status
|
||||
* even NOMEM error is encountered
|
||||
*/
|
||||
if(!err_data->err_addr)
|
||||
dev_warn(adev->dev, "Failed to alloc memory for "
|
||||
"umc error address record!\n");
|
||||
/* still call query_ras_error_address to clear error status
|
||||
* even NOMEM error is encountered
|
||||
*/
|
||||
if(!err_data->err_addr)
|
||||
dev_warn(adev->dev, "Failed to alloc memory for "
|
||||
"umc error address record!\n");
|
||||
|
||||
/* umc query_ras_error_address is also responsible for clearing
|
||||
* error status
|
||||
*/
|
||||
adev->umc.ras_funcs->query_ras_error_address(adev, ras_error_status);
|
||||
/* umc query_ras_error_address is also responsible for clearing
|
||||
* error status
|
||||
*/
|
||||
adev->umc.ras_funcs->query_ras_error_address(adev, ras_error_status);
|
||||
}
|
||||
} else if (!ret) {
|
||||
if (adev->umc.ras_funcs &&
|
||||
adev->umc.ras_funcs->ecc_info_query_ras_error_count)
|
||||
adev->umc.ras_funcs->ecc_info_query_ras_error_count(adev, ras_error_status);
|
||||
|
||||
if (adev->umc.ras_funcs &&
|
||||
adev->umc.ras_funcs->ecc_info_query_ras_error_address &&
|
||||
adev->umc.max_ras_err_cnt_per_query) {
|
||||
err_data->err_addr =
|
||||
kcalloc(adev->umc.max_ras_err_cnt_per_query,
|
||||
sizeof(struct eeprom_table_record), GFP_KERNEL);
|
||||
|
||||
/* still call query_ras_error_address to clear error status
|
||||
* even NOMEM error is encountered
|
||||
*/
|
||||
if(!err_data->err_addr)
|
||||
dev_warn(adev->dev, "Failed to alloc memory for "
|
||||
"umc error address record!\n");
|
||||
|
||||
/* umc query_ras_error_address is also responsible for clearing
|
||||
* error status
|
||||
*/
|
||||
adev->umc.ras_funcs->ecc_info_query_ras_error_address(adev, ras_error_status);
|
||||
}
|
||||
}
|
||||
|
||||
/* only uncorrectable error needs gpu reset */
|
||||
|
||||
@ -49,6 +49,10 @@ struct amdgpu_umc_ras_funcs {
|
||||
void (*query_ras_error_address)(struct amdgpu_device *adev,
|
||||
void *ras_error_status);
|
||||
bool (*query_ras_poison_mode)(struct amdgpu_device *adev);
|
||||
void (*ecc_info_query_ras_error_count)(struct amdgpu_device *adev,
|
||||
void *ras_error_status);
|
||||
void (*ecc_info_query_ras_error_address)(struct amdgpu_device *adev,
|
||||
void *ras_error_status);
|
||||
};
|
||||
|
||||
struct amdgpu_umc_funcs {
|
||||
|
||||
@ -283,17 +283,15 @@ static int amdgpu_virt_init_ras_err_handler_data(struct amdgpu_device *adev)
|
||||
|
||||
*data = kmalloc(sizeof(struct amdgpu_virt_ras_err_handler_data), GFP_KERNEL);
|
||||
if (!*data)
|
||||
return -ENOMEM;
|
||||
goto data_failure;
|
||||
|
||||
bps = kmalloc_array(align_space, sizeof((*data)->bps), GFP_KERNEL);
|
||||
bps_bo = kmalloc_array(align_space, sizeof((*data)->bps_bo), GFP_KERNEL);
|
||||
if (!bps)
|
||||
goto bps_failure;
|
||||
|
||||
if (!bps || !bps_bo) {
|
||||
kfree(bps);
|
||||
kfree(bps_bo);
|
||||
kfree(*data);
|
||||
return -ENOMEM;
|
||||
}
|
||||
bps_bo = kmalloc_array(align_space, sizeof((*data)->bps_bo), GFP_KERNEL);
|
||||
if (!bps_bo)
|
||||
goto bps_bo_failure;
|
||||
|
||||
(*data)->bps = bps;
|
||||
(*data)->bps_bo = bps_bo;
|
||||
@ -303,6 +301,13 @@ static int amdgpu_virt_init_ras_err_handler_data(struct amdgpu_device *adev)
|
||||
virt->ras_init_done = true;
|
||||
|
||||
return 0;
|
||||
|
||||
bps_bo_failure:
|
||||
kfree(bps);
|
||||
bps_failure:
|
||||
kfree(*data);
|
||||
data_failure:
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
static void amdgpu_virt_ras_release_bp(struct amdgpu_device *adev)
|
||||
|
||||
@ -2092,22 +2092,18 @@ static int dce_v8_0_pick_dig_encoder(struct drm_encoder *encoder)
|
||||
return 1;
|
||||
else
|
||||
return 0;
|
||||
break;
|
||||
case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1:
|
||||
if (dig->linkb)
|
||||
return 3;
|
||||
else
|
||||
return 2;
|
||||
break;
|
||||
case ENCODER_OBJECT_ID_INTERNAL_UNIPHY2:
|
||||
if (dig->linkb)
|
||||
return 5;
|
||||
else
|
||||
return 4;
|
||||
break;
|
||||
case ENCODER_OBJECT_ID_INTERNAL_UNIPHY3:
|
||||
return 6;
|
||||
break;
|
||||
default:
|
||||
DRM_ERROR("invalid encoder_id: 0x%x\n", amdgpu_encoder->encoder_id);
|
||||
return 0;
|
||||
|
||||
@ -7707,8 +7707,19 @@ static uint64_t gfx_v10_0_get_gpu_clock_counter(struct amdgpu_device *adev)
|
||||
switch (adev->ip_versions[GC_HWIP][0]) {
|
||||
case IP_VERSION(10, 3, 1):
|
||||
case IP_VERSION(10, 3, 3):
|
||||
clock = (uint64_t)RREG32_SOC15(SMUIO, 0, mmGOLDEN_TSC_COUNT_LOWER_Vangogh) |
|
||||
((uint64_t)RREG32_SOC15(SMUIO, 0, mmGOLDEN_TSC_COUNT_UPPER_Vangogh) << 32ULL);
|
||||
preempt_disable();
|
||||
clock_hi = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_UPPER_Vangogh);
|
||||
clock_lo = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_LOWER_Vangogh);
|
||||
hi_check = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_UPPER_Vangogh);
|
||||
/* The SMUIO TSC clock frequency is 100MHz, which sets 32-bit carry over
|
||||
* roughly every 42 seconds.
|
||||
*/
|
||||
if (hi_check != clock_hi) {
|
||||
clock_lo = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_LOWER_Vangogh);
|
||||
clock_hi = hi_check;
|
||||
}
|
||||
preempt_enable();
|
||||
clock = clock_lo | (clock_hi << 32ULL);
|
||||
break;
|
||||
default:
|
||||
preempt_disable();
|
||||
|
||||
@ -140,6 +140,11 @@ MODULE_FIRMWARE("amdgpu/aldebaran_rlc.bin");
|
||||
#define mmTCP_CHAN_STEER_5_ARCT 0x0b0c
|
||||
#define mmTCP_CHAN_STEER_5_ARCT_BASE_IDX 0
|
||||
|
||||
#define mmGOLDEN_TSC_COUNT_UPPER_Renoir 0x0025
|
||||
#define mmGOLDEN_TSC_COUNT_UPPER_Renoir_BASE_IDX 1
|
||||
#define mmGOLDEN_TSC_COUNT_LOWER_Renoir 0x0026
|
||||
#define mmGOLDEN_TSC_COUNT_LOWER_Renoir_BASE_IDX 1
|
||||
|
||||
enum ta_ras_gfx_subblock {
|
||||
/*CPC*/
|
||||
TA_RAS_BLOCK__GFX_CPC_INDEX_START = 0,
|
||||
@ -4238,19 +4243,38 @@ failed_kiq_read:
|
||||
|
||||
static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev)
|
||||
{
|
||||
uint64_t clock;
|
||||
uint64_t clock, clock_lo, clock_hi, hi_check;
|
||||
|
||||
amdgpu_gfx_off_ctrl(adev, false);
|
||||
mutex_lock(&adev->gfx.gpu_clock_mutex);
|
||||
if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 0, 1) && amdgpu_sriov_runtime(adev)) {
|
||||
clock = gfx_v9_0_kiq_read_clock(adev);
|
||||
} else {
|
||||
WREG32_SOC15(GC, 0, mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1);
|
||||
clock = (uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_LSB) |
|
||||
((uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
|
||||
switch (adev->ip_versions[GC_HWIP][0]) {
|
||||
case IP_VERSION(9, 3, 0):
|
||||
preempt_disable();
|
||||
clock_hi = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_UPPER_Renoir);
|
||||
clock_lo = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_LOWER_Renoir);
|
||||
hi_check = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_UPPER_Renoir);
|
||||
/* The SMUIO TSC clock frequency is 100MHz, which sets 32-bit carry over
|
||||
* roughly every 42 seconds.
|
||||
*/
|
||||
if (hi_check != clock_hi) {
|
||||
clock_lo = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_LOWER_Renoir);
|
||||
clock_hi = hi_check;
|
||||
}
|
||||
preempt_enable();
|
||||
clock = clock_lo | (clock_hi << 32ULL);
|
||||
break;
|
||||
default:
|
||||
amdgpu_gfx_off_ctrl(adev, false);
|
||||
mutex_lock(&adev->gfx.gpu_clock_mutex);
|
||||
if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 0, 1) && amdgpu_sriov_runtime(adev)) {
|
||||
clock = gfx_v9_0_kiq_read_clock(adev);
|
||||
} else {
|
||||
WREG32_SOC15(GC, 0, mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1);
|
||||
clock = (uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_LSB) |
|
||||
((uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
|
||||
}
|
||||
mutex_unlock(&adev->gfx.gpu_clock_mutex);
|
||||
amdgpu_gfx_off_ctrl(adev, true);
|
||||
break;
|
||||
}
|
||||
mutex_unlock(&adev->gfx.gpu_clock_mutex);
|
||||
amdgpu_gfx_off_ctrl(adev, true);
|
||||
return clock;
|
||||
}
|
||||
|
||||
|
||||
@ -160,6 +160,7 @@ static int navi10_ih_toggle_ring_interrupts(struct amdgpu_device *adev,
|
||||
|
||||
tmp = RREG32(ih_regs->ih_rb_cntl);
|
||||
tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RB_ENABLE, (enable ? 1 : 0));
|
||||
tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RB_GPU_TS_ENABLE, 1);
|
||||
/* enable_intr field is only valid in ring0 */
|
||||
if (ih == &adev->irq.ih)
|
||||
tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, ENABLE_INTR, (enable ? 1 : 0));
|
||||
@ -275,10 +276,8 @@ static int navi10_ih_enable_ring(struct amdgpu_device *adev,
|
||||
tmp = navi10_ih_rb_cntl(ih, tmp);
|
||||
if (ih == &adev->irq.ih)
|
||||
tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RPTR_REARM, !!adev->irq.msi_enabled);
|
||||
if (ih == &adev->irq.ih1) {
|
||||
tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_ENABLE, 0);
|
||||
if (ih == &adev->irq.ih1)
|
||||
tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RB_FULL_DRAIN_ENABLE, 1);
|
||||
}
|
||||
|
||||
if (amdgpu_sriov_vf(adev) && amdgpu_sriov_reg_indirect_ih(adev)) {
|
||||
if (psp_reg_program(&adev->psp, ih_regs->psp_reg_id, tmp)) {
|
||||
@ -319,7 +318,6 @@ static int navi10_ih_irq_init(struct amdgpu_device *adev)
|
||||
{
|
||||
struct amdgpu_ih_ring *ih[] = {&adev->irq.ih, &adev->irq.ih1, &adev->irq.ih2};
|
||||
u32 ih_chicken;
|
||||
u32 tmp;
|
||||
int ret;
|
||||
int i;
|
||||
|
||||
@ -363,15 +361,6 @@ static int navi10_ih_irq_init(struct amdgpu_device *adev)
|
||||
adev->nbio.funcs->ih_doorbell_range(adev, ih[0]->use_doorbell,
|
||||
ih[0]->doorbell_index);
|
||||
|
||||
tmp = RREG32_SOC15(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL);
|
||||
tmp = REG_SET_FIELD(tmp, IH_STORM_CLIENT_LIST_CNTL,
|
||||
CLIENT18_IS_STORM_CLIENT, 1);
|
||||
WREG32_SOC15(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL, tmp);
|
||||
|
||||
tmp = RREG32_SOC15(OSSSYS, 0, mmIH_INT_FLOOD_CNTL);
|
||||
tmp = REG_SET_FIELD(tmp, IH_INT_FLOOD_CNTL, FLOOD_CNTL_ENABLE, 1);
|
||||
WREG32_SOC15(OSSSYS, 0, mmIH_INT_FLOOD_CNTL, tmp);
|
||||
|
||||
pci_set_master(adev->pdev);
|
||||
|
||||
/* enable interrupts */
|
||||
@ -420,12 +409,19 @@ static u32 navi10_ih_get_wptr(struct amdgpu_device *adev,
|
||||
u32 wptr, tmp;
|
||||
struct amdgpu_ih_regs *ih_regs;
|
||||
|
||||
wptr = le32_to_cpu(*ih->wptr_cpu);
|
||||
if (ih == &adev->irq.ih) {
|
||||
/* Only ring0 supports writeback. On other rings fall back
|
||||
* to register-based code with overflow checking below.
|
||||
*/
|
||||
wptr = le32_to_cpu(*ih->wptr_cpu);
|
||||
|
||||
if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW))
|
||||
goto out;
|
||||
}
|
||||
|
||||
ih_regs = &ih->ih_regs;
|
||||
|
||||
if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW))
|
||||
goto out;
|
||||
|
||||
/* Double check that the overflow wasn't already cleared. */
|
||||
wptr = RREG32_NO_KIQ(ih_regs->ih_rb_wptr);
|
||||
if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW))
|
||||
goto out;
|
||||
@ -513,15 +509,11 @@ static int navi10_ih_self_irq(struct amdgpu_device *adev,
|
||||
struct amdgpu_irq_src *source,
|
||||
struct amdgpu_iv_entry *entry)
|
||||
{
|
||||
uint32_t wptr = cpu_to_le32(entry->src_data[0]);
|
||||
|
||||
switch (entry->ring_id) {
|
||||
case 1:
|
||||
*adev->irq.ih1.wptr_cpu = wptr;
|
||||
schedule_work(&adev->irq.ih1_work);
|
||||
break;
|
||||
case 2:
|
||||
*adev->irq.ih2.wptr_cpu = wptr;
|
||||
schedule_work(&adev->irq.ih2_work);
|
||||
break;
|
||||
default: break;
|
||||
|
||||
@ -359,6 +359,10 @@ static void nbio_v2_3_init_registers(struct amdgpu_device *adev)
|
||||
|
||||
if (def != data)
|
||||
WREG32_PCIE(smnPCIE_CONFIG_CNTL, data);
|
||||
|
||||
if (amdgpu_sriov_vf(adev))
|
||||
adev->rmmio_remap.reg_offset = SOC15_REG_OFFSET(NBIO, 0,
|
||||
mmBIF_BX_DEV0_EPF0_VF0_HDP_MEM_COHERENCY_FLUSH_CNTL) << 2;
|
||||
}
|
||||
|
||||
#define NAVI10_PCIE__LC_L0S_INACTIVITY_DEFAULT 0x00000000 // off by default, no gains over L1
|
||||
|
||||
@ -276,6 +276,10 @@ static void nbio_v6_1_init_registers(struct amdgpu_device *adev)
|
||||
|
||||
if (def != data)
|
||||
WREG32_PCIE(smnPCIE_CI_CNTL, data);
|
||||
|
||||
if (amdgpu_sriov_vf(adev))
|
||||
adev->rmmio_remap.reg_offset = SOC15_REG_OFFSET(NBIO, 0,
|
||||
mmBIF_BX_DEV0_EPF0_VF0_HDP_MEM_COHERENCY_FLUSH_CNTL) << 2;
|
||||
}
|
||||
|
||||
static void nbio_v6_1_program_ltr(struct amdgpu_device *adev)
|
||||
|
||||
@ -273,7 +273,9 @@ const struct nbio_hdp_flush_reg nbio_v7_0_hdp_flush_reg = {
|
||||
|
||||
static void nbio_v7_0_init_registers(struct amdgpu_device *adev)
|
||||
{
|
||||
|
||||
if (amdgpu_sriov_vf(adev))
|
||||
adev->rmmio_remap.reg_offset =
|
||||
SOC15_REG_OFFSET(NBIO, 0, mmHDP_MEM_COHERENCY_FLUSH_CNTL) << 2;
|
||||
}
|
||||
|
||||
const struct amdgpu_nbio_funcs nbio_v7_0_funcs = {
|
||||
|
||||
@ -371,6 +371,10 @@ static void nbio_v7_2_init_registers(struct amdgpu_device *adev)
|
||||
if (def != data)
|
||||
WREG32_PCIE_PORT(SOC15_REG_OFFSET(NBIO, 0, regPCIE_CONFIG_CNTL), data);
|
||||
}
|
||||
|
||||
if (amdgpu_sriov_vf(adev))
|
||||
adev->rmmio_remap.reg_offset = SOC15_REG_OFFSET(NBIO, 0,
|
||||
regBIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_CNTL) << 2;
|
||||
}
|
||||
|
||||
const struct amdgpu_nbio_funcs nbio_v7_2_funcs = {
|
||||
|
||||
@ -362,7 +362,9 @@ const struct nbio_hdp_flush_reg nbio_v7_4_hdp_flush_reg_ald = {
|
||||
|
||||
static void nbio_v7_4_init_registers(struct amdgpu_device *adev)
|
||||
{
|
||||
|
||||
if (amdgpu_sriov_vf(adev))
|
||||
adev->rmmio_remap.reg_offset = SOC15_REG_OFFSET(NBIO, 0,
|
||||
mmBIF_BX_DEV0_EPF0_VF0_HDP_MEM_COHERENCY_FLUSH_CNTL) << 2;
|
||||
}
|
||||
|
||||
static void nbio_v7_4_handle_ras_controller_intr_no_bifring(struct amdgpu_device *adev)
|
||||
@ -692,6 +694,9 @@ static void nbio_v7_4_program_aspm(struct amdgpu_device *adev)
|
||||
{
|
||||
uint32_t def, data;
|
||||
|
||||
if (adev->ip_versions[NBIO_HWIP][0] == IP_VERSION(7, 4, 4))
|
||||
return;
|
||||
|
||||
def = data = RREG32_PCIE(smnPCIE_LC_CNTL);
|
||||
data &= ~PCIE_LC_CNTL__LC_L1_INACTIVITY_MASK;
|
||||
data &= ~PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK;
|
||||
|
||||
@ -731,8 +731,10 @@ static int nv_common_early_init(void *handle)
|
||||
#define MMIO_REG_HOLE_OFFSET (0x80000 - PAGE_SIZE)
|
||||
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
|
||||
|
||||
adev->rmmio_remap.reg_offset = MMIO_REG_HOLE_OFFSET;
|
||||
adev->rmmio_remap.bus_addr = adev->rmmio_base + MMIO_REG_HOLE_OFFSET;
|
||||
if (!amdgpu_sriov_vf(adev)) {
|
||||
adev->rmmio_remap.reg_offset = MMIO_REG_HOLE_OFFSET;
|
||||
adev->rmmio_remap.bus_addr = adev->rmmio_base + MMIO_REG_HOLE_OFFSET;
|
||||
}
|
||||
adev->smc_rreg = NULL;
|
||||
adev->smc_wreg = NULL;
|
||||
adev->pcie_rreg = &nv_pcie_rreg;
|
||||
@ -1032,7 +1034,7 @@ static int nv_common_hw_init(void *handle)
|
||||
* for the purpose of expose those registers
|
||||
* to process space
|
||||
*/
|
||||
if (adev->nbio.funcs->remap_hdp_registers)
|
||||
if (adev->nbio.funcs->remap_hdp_registers && !amdgpu_sriov_vf(adev))
|
||||
adev->nbio.funcs->remap_hdp_registers(adev);
|
||||
/* enable the doorbell aperture */
|
||||
nv_enable_doorbell_aperture(adev, true);
|
||||
|
||||
@ -971,8 +971,10 @@ static int soc15_common_early_init(void *handle)
|
||||
#define MMIO_REG_HOLE_OFFSET (0x80000 - PAGE_SIZE)
|
||||
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
|
||||
|
||||
adev->rmmio_remap.reg_offset = MMIO_REG_HOLE_OFFSET;
|
||||
adev->rmmio_remap.bus_addr = adev->rmmio_base + MMIO_REG_HOLE_OFFSET;
|
||||
if (!amdgpu_sriov_vf(adev)) {
|
||||
adev->rmmio_remap.reg_offset = MMIO_REG_HOLE_OFFSET;
|
||||
adev->rmmio_remap.bus_addr = adev->rmmio_base + MMIO_REG_HOLE_OFFSET;
|
||||
}
|
||||
adev->smc_rreg = NULL;
|
||||
adev->smc_wreg = NULL;
|
||||
adev->pcie_rreg = &soc15_pcie_rreg;
|
||||
@ -1285,7 +1287,7 @@ static int soc15_common_hw_init(void *handle)
|
||||
* for the purpose of expose those registers
|
||||
* to process space
|
||||
*/
|
||||
if (adev->nbio.funcs->remap_hdp_registers)
|
||||
if (adev->nbio.funcs->remap_hdp_registers && !amdgpu_sriov_vf(adev))
|
||||
adev->nbio.funcs->remap_hdp_registers(adev);
|
||||
|
||||
/* enable the doorbell aperture */
|
||||
|
||||
@ -50,6 +50,165 @@ static inline uint32_t get_umc_v6_7_reg_offset(struct amdgpu_device *adev,
|
||||
return adev->umc.channel_offs * ch_inst + UMC_V6_7_INST_DIST * umc_inst;
|
||||
}
|
||||
|
||||
static inline uint32_t get_umc_v6_7_channel_index(struct amdgpu_device *adev,
|
||||
uint32_t umc_inst,
|
||||
uint32_t ch_inst)
|
||||
{
|
||||
return adev->umc.channel_idx_tbl[umc_inst * adev->umc.channel_inst_num + ch_inst];
|
||||
}
|
||||
|
||||
static void umc_v6_7_ecc_info_query_correctable_error_count(struct amdgpu_device *adev,
|
||||
uint32_t channel_index,
|
||||
unsigned long *error_count)
|
||||
{
|
||||
uint32_t ecc_err_cnt;
|
||||
uint64_t mc_umc_status;
|
||||
struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
|
||||
|
||||
/*
|
||||
* select the lower chip and check the error count
|
||||
* skip add error count, calc error counter only from mca_umc_status
|
||||
*/
|
||||
ecc_err_cnt = ras->umc_ecc.ecc[channel_index].ce_count_lo_chip;
|
||||
|
||||
/*
|
||||
* select the higher chip and check the err counter
|
||||
* skip add error count, calc error counter only from mca_umc_status
|
||||
*/
|
||||
ecc_err_cnt = ras->umc_ecc.ecc[channel_index].ce_count_hi_chip;
|
||||
|
||||
/* check for SRAM correctable error
|
||||
MCUMC_STATUS is a 64 bit register */
|
||||
mc_umc_status = ras->umc_ecc.ecc[channel_index].mca_umc_status;
|
||||
if (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Val) == 1 &&
|
||||
REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, CECC) == 1)
|
||||
*error_count += 1;
|
||||
}
|
||||
|
||||
static void umc_v6_7_ecc_info_querry_uncorrectable_error_count(struct amdgpu_device *adev,
|
||||
uint32_t channel_index,
|
||||
unsigned long *error_count)
|
||||
{
|
||||
uint64_t mc_umc_status;
|
||||
struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
|
||||
|
||||
/* check the MCUMC_STATUS */
|
||||
mc_umc_status = ras->umc_ecc.ecc[channel_index].mca_umc_status;
|
||||
if ((REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Val) == 1) &&
|
||||
(REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Deferred) == 1 ||
|
||||
REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UECC) == 1 ||
|
||||
REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, PCC) == 1 ||
|
||||
REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UC) == 1 ||
|
||||
REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, TCC) == 1))
|
||||
*error_count += 1;
|
||||
}
|
||||
|
||||
static void umc_v6_7_ecc_info_query_ras_error_count(struct amdgpu_device *adev,
|
||||
void *ras_error_status)
|
||||
{
|
||||
struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
|
||||
|
||||
uint32_t umc_inst = 0;
|
||||
uint32_t ch_inst = 0;
|
||||
uint32_t umc_reg_offset = 0;
|
||||
uint32_t channel_index = 0;
|
||||
|
||||
/*TODO: driver needs to toggle DF Cstate to ensure
|
||||
* safe access of UMC registers. Will add the protection */
|
||||
LOOP_UMC_INST_AND_CH(umc_inst, ch_inst) {
|
||||
umc_reg_offset = get_umc_v6_7_reg_offset(adev,
|
||||
umc_inst,
|
||||
ch_inst);
|
||||
channel_index = get_umc_v6_7_channel_index(adev,
|
||||
umc_inst,
|
||||
ch_inst);
|
||||
umc_v6_7_ecc_info_query_correctable_error_count(adev,
|
||||
channel_index,
|
||||
&(err_data->ce_count));
|
||||
umc_v6_7_ecc_info_querry_uncorrectable_error_count(adev,
|
||||
channel_index,
|
||||
&(err_data->ue_count));
|
||||
}
|
||||
}
|
||||
|
||||
static void umc_v6_7_ecc_info_query_error_address(struct amdgpu_device *adev,
|
||||
struct ras_err_data *err_data,
|
||||
uint32_t umc_reg_offset,
|
||||
uint32_t ch_inst,
|
||||
uint32_t umc_inst)
|
||||
{
|
||||
uint64_t mc_umc_status, err_addr, retired_page;
|
||||
struct eeprom_table_record *err_rec;
|
||||
uint32_t channel_index;
|
||||
struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
|
||||
|
||||
channel_index =
|
||||
adev->umc.channel_idx_tbl[umc_inst * adev->umc.channel_inst_num + ch_inst];
|
||||
|
||||
mc_umc_status = ras->umc_ecc.ecc[channel_index].mca_umc_status;
|
||||
|
||||
if (mc_umc_status == 0)
|
||||
return;
|
||||
|
||||
if (!err_data->err_addr)
|
||||
return;
|
||||
|
||||
err_rec = &err_data->err_addr[err_data->err_addr_cnt];
|
||||
|
||||
/* calculate error address if ue/ce error is detected */
|
||||
if (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Val) == 1 &&
|
||||
(REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UECC) == 1 ||
|
||||
REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, CECC) == 1)) {
|
||||
|
||||
err_addr = ras->umc_ecc.ecc[channel_index].mca_umc_addr;
|
||||
err_addr = REG_GET_FIELD(err_addr, MCA_UMC_UMC0_MCUMC_ADDRT0, ErrorAddr);
|
||||
|
||||
/* translate umc channel address to soc pa, 3 parts are included */
|
||||
retired_page = ADDR_OF_8KB_BLOCK(err_addr) |
|
||||
ADDR_OF_256B_BLOCK(channel_index) |
|
||||
OFFSET_IN_256B_BLOCK(err_addr);
|
||||
|
||||
/* we only save ue error information currently, ce is skipped */
|
||||
if (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UECC)
|
||||
== 1) {
|
||||
err_rec->address = err_addr;
|
||||
/* page frame address is saved */
|
||||
err_rec->retired_page = retired_page >> AMDGPU_GPU_PAGE_SHIFT;
|
||||
err_rec->ts = (uint64_t)ktime_get_real_seconds();
|
||||
err_rec->err_type = AMDGPU_RAS_EEPROM_ERR_NON_RECOVERABLE;
|
||||
err_rec->cu = 0;
|
||||
err_rec->mem_channel = channel_index;
|
||||
err_rec->mcumc_id = umc_inst;
|
||||
|
||||
err_data->err_addr_cnt++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void umc_v6_7_ecc_info_query_ras_error_address(struct amdgpu_device *adev,
|
||||
void *ras_error_status)
|
||||
{
|
||||
struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
|
||||
|
||||
uint32_t umc_inst = 0;
|
||||
uint32_t ch_inst = 0;
|
||||
uint32_t umc_reg_offset = 0;
|
||||
|
||||
/*TODO: driver needs to toggle DF Cstate to ensure
|
||||
* safe access of UMC resgisters. Will add the protection
|
||||
* when firmware interface is ready */
|
||||
LOOP_UMC_INST_AND_CH(umc_inst, ch_inst) {
|
||||
umc_reg_offset = get_umc_v6_7_reg_offset(adev,
|
||||
umc_inst,
|
||||
ch_inst);
|
||||
umc_v6_7_ecc_info_query_error_address(adev,
|
||||
err_data,
|
||||
umc_reg_offset,
|
||||
ch_inst,
|
||||
umc_inst);
|
||||
}
|
||||
}
|
||||
|
||||
static void umc_v6_7_query_correctable_error_count(struct amdgpu_device *adev,
|
||||
uint32_t umc_reg_offset,
|
||||
unsigned long *error_count)
|
||||
@ -327,4 +486,6 @@ const struct amdgpu_umc_ras_funcs umc_v6_7_ras_funcs = {
|
||||
.query_ras_error_count = umc_v6_7_query_ras_error_count,
|
||||
.query_ras_error_address = umc_v6_7_query_ras_error_address,
|
||||
.query_ras_poison_mode = umc_v6_7_query_ras_poison_mode,
|
||||
.ecc_info_query_ras_error_count = umc_v6_7_ecc_info_query_ras_error_count,
|
||||
.ecc_info_query_ras_error_address = umc_v6_7_ecc_info_query_ras_error_address,
|
||||
};
|
||||
|
||||
@ -43,15 +43,15 @@ static bool cik_event_interrupt_isr(struct kfd_dev *dev,
|
||||
*/
|
||||
if ((ihre->source_id == CIK_INTSRC_GFX_PAGE_INV_FAULT ||
|
||||
ihre->source_id == CIK_INTSRC_GFX_MEM_PROT_FAULT) &&
|
||||
dev->device_info->asic_family == CHIP_HAWAII) {
|
||||
dev->adev->asic_type == CHIP_HAWAII) {
|
||||
struct cik_ih_ring_entry *tmp_ihre =
|
||||
(struct cik_ih_ring_entry *)patched_ihre;
|
||||
|
||||
*patched_flag = true;
|
||||
*tmp_ihre = *ihre;
|
||||
|
||||
vmid = f2g->read_vmid_from_vmfault_reg(dev->kgd);
|
||||
ret = f2g->get_atc_vmid_pasid_mapping_info(dev->kgd, vmid, &pasid);
|
||||
vmid = f2g->read_vmid_from_vmfault_reg(dev->adev);
|
||||
ret = f2g->get_atc_vmid_pasid_mapping_info(dev->adev, vmid, &pasid);
|
||||
|
||||
tmp_ihre->ring_id &= 0x000000ff;
|
||||
tmp_ihre->ring_id |= vmid << 8;
|
||||
@ -113,7 +113,7 @@ static void cik_event_interrupt_wq(struct kfd_dev *dev,
|
||||
kfd_process_vm_fault(dev->dqm, pasid);
|
||||
|
||||
memset(&info, 0, sizeof(info));
|
||||
amdgpu_amdkfd_gpuvm_get_vm_fault_info(dev->kgd, &info);
|
||||
amdgpu_amdkfd_gpuvm_get_vm_fault_info(dev->adev, &info);
|
||||
if (!info.page_addr && !info.status)
|
||||
return;
|
||||
|
||||
|
||||
@ -321,7 +321,7 @@ static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p,
|
||||
/* Return gpu_id as doorbell offset for mmap usage */
|
||||
args->doorbell_offset = KFD_MMAP_TYPE_DOORBELL;
|
||||
args->doorbell_offset |= KFD_MMAP_GPU_ID(args->gpu_id);
|
||||
if (KFD_IS_SOC15(dev->device_info->asic_family))
|
||||
if (KFD_IS_SOC15(dev))
|
||||
/* On SOC15 ASICs, include the doorbell offset within the
|
||||
* process doorbell frame, which is 2 pages.
|
||||
*/
|
||||
@ -580,7 +580,7 @@ static int kfd_ioctl_dbg_register(struct file *filep,
|
||||
if (!dev)
|
||||
return -EINVAL;
|
||||
|
||||
if (dev->device_info->asic_family == CHIP_CARRIZO) {
|
||||
if (dev->adev->asic_type == CHIP_CARRIZO) {
|
||||
pr_debug("kfd_ioctl_dbg_register not supported on CZ\n");
|
||||
return -EINVAL;
|
||||
}
|
||||
@ -631,7 +631,7 @@ static int kfd_ioctl_dbg_unregister(struct file *filep,
|
||||
if (!dev || !dev->dbgmgr)
|
||||
return -EINVAL;
|
||||
|
||||
if (dev->device_info->asic_family == CHIP_CARRIZO) {
|
||||
if (dev->adev->asic_type == CHIP_CARRIZO) {
|
||||
pr_debug("kfd_ioctl_dbg_unregister not supported on CZ\n");
|
||||
return -EINVAL;
|
||||
}
|
||||
@ -676,7 +676,7 @@ static int kfd_ioctl_dbg_address_watch(struct file *filep,
|
||||
if (!dev)
|
||||
return -EINVAL;
|
||||
|
||||
if (dev->device_info->asic_family == CHIP_CARRIZO) {
|
||||
if (dev->adev->asic_type == CHIP_CARRIZO) {
|
||||
pr_debug("kfd_ioctl_dbg_wave_control not supported on CZ\n");
|
||||
return -EINVAL;
|
||||
}
|
||||
@ -784,7 +784,7 @@ static int kfd_ioctl_dbg_wave_control(struct file *filep,
|
||||
if (!dev)
|
||||
return -EINVAL;
|
||||
|
||||
if (dev->device_info->asic_family == CHIP_CARRIZO) {
|
||||
if (dev->adev->asic_type == CHIP_CARRIZO) {
|
||||
pr_debug("kfd_ioctl_dbg_wave_control not supported on CZ\n");
|
||||
return -EINVAL;
|
||||
}
|
||||
@ -851,7 +851,7 @@ static int kfd_ioctl_get_clock_counters(struct file *filep,
|
||||
dev = kfd_device_by_id(args->gpu_id);
|
||||
if (dev)
|
||||
/* Reading GPU clock counter from KGD */
|
||||
args->gpu_clock_counter = amdgpu_amdkfd_get_gpu_clock_counter(dev->kgd);
|
||||
args->gpu_clock_counter = amdgpu_amdkfd_get_gpu_clock_counter(dev->adev);
|
||||
else
|
||||
/* Node without GPU resource */
|
||||
args->gpu_clock_counter = 0;
|
||||
@ -1041,7 +1041,7 @@ static int kfd_ioctl_create_event(struct file *filp, struct kfd_process *p,
|
||||
goto out_unlock;
|
||||
}
|
||||
|
||||
err = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(kfd->kgd,
|
||||
err = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(kfd->adev,
|
||||
mem, &kern_addr, &size);
|
||||
if (err) {
|
||||
pr_err("Failed to map event page to kernel\n");
|
||||
@ -1051,7 +1051,7 @@ static int kfd_ioctl_create_event(struct file *filp, struct kfd_process *p,
|
||||
err = kfd_event_page_set(p, kern_addr, size);
|
||||
if (err) {
|
||||
pr_err("Failed to set event page\n");
|
||||
amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(kfd->kgd, mem);
|
||||
amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(kfd->adev, mem);
|
||||
goto out_unlock;
|
||||
}
|
||||
|
||||
@ -1137,7 +1137,7 @@ static int kfd_ioctl_set_scratch_backing_va(struct file *filep,
|
||||
if (dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS &&
|
||||
pdd->qpd.vmid != 0 && dev->kfd2kgd->set_scratch_backing_va)
|
||||
dev->kfd2kgd->set_scratch_backing_va(
|
||||
dev->kgd, args->va_addr, pdd->qpd.vmid);
|
||||
dev->adev, args->va_addr, pdd->qpd.vmid);
|
||||
|
||||
return 0;
|
||||
|
||||
@ -1158,7 +1158,7 @@ static int kfd_ioctl_get_tile_config(struct file *filep,
|
||||
if (!dev)
|
||||
return -EINVAL;
|
||||
|
||||
amdgpu_amdkfd_get_tile_config(dev->kgd, &config);
|
||||
amdgpu_amdkfd_get_tile_config(dev->adev, &config);
|
||||
|
||||
args->gb_addr_config = config.gb_addr_config;
|
||||
args->num_banks = config.num_banks;
|
||||
@ -1244,7 +1244,7 @@ bool kfd_dev_is_large_bar(struct kfd_dev *dev)
|
||||
if (dev->use_iommu_v2)
|
||||
return false;
|
||||
|
||||
amdgpu_amdkfd_get_local_mem_info(dev->kgd, &mem_info);
|
||||
amdgpu_amdkfd_get_local_mem_info(dev->adev, &mem_info);
|
||||
if (mem_info.local_mem_size_private == 0 &&
|
||||
mem_info.local_mem_size_public > 0)
|
||||
return true;
|
||||
@ -1313,7 +1313,7 @@ static int kfd_ioctl_alloc_memory_of_gpu(struct file *filep,
|
||||
err = -EINVAL;
|
||||
goto err_unlock;
|
||||
}
|
||||
offset = amdgpu_amdkfd_get_mmio_remap_phys_addr(dev->kgd);
|
||||
offset = dev->adev->rmmio_remap.bus_addr;
|
||||
if (!offset) {
|
||||
err = -ENOMEM;
|
||||
goto err_unlock;
|
||||
@ -1321,7 +1321,7 @@ static int kfd_ioctl_alloc_memory_of_gpu(struct file *filep,
|
||||
}
|
||||
|
||||
err = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
|
||||
dev->kgd, args->va_addr, args->size,
|
||||
dev->adev, args->va_addr, args->size,
|
||||
pdd->drm_priv, (struct kgd_mem **) &mem, &offset,
|
||||
flags);
|
||||
|
||||
@ -1353,7 +1353,7 @@ static int kfd_ioctl_alloc_memory_of_gpu(struct file *filep,
|
||||
return 0;
|
||||
|
||||
err_free:
|
||||
amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->kgd, (struct kgd_mem *)mem,
|
||||
amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->adev, (struct kgd_mem *)mem,
|
||||
pdd->drm_priv, NULL);
|
||||
err_unlock:
|
||||
mutex_unlock(&p->mutex);
|
||||
@ -1399,7 +1399,7 @@ static int kfd_ioctl_free_memory_of_gpu(struct file *filep,
|
||||
goto err_unlock;
|
||||
}
|
||||
|
||||
ret = amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->kgd,
|
||||
ret = amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->adev,
|
||||
(struct kgd_mem *)mem, pdd->drm_priv, &size);
|
||||
|
||||
/* If freeing the buffer failed, leave the handle in place for
|
||||
@ -1484,7 +1484,7 @@ static int kfd_ioctl_map_memory_to_gpu(struct file *filep,
|
||||
goto get_mem_obj_from_handle_failed;
|
||||
}
|
||||
err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
|
||||
peer->kgd, (struct kgd_mem *)mem,
|
||||
peer->adev, (struct kgd_mem *)mem,
|
||||
peer_pdd->drm_priv, &table_freed);
|
||||
if (err) {
|
||||
pr_err("Failed to map to gpu %d/%d\n",
|
||||
@ -1496,7 +1496,7 @@ static int kfd_ioctl_map_memory_to_gpu(struct file *filep,
|
||||
|
||||
mutex_unlock(&p->mutex);
|
||||
|
||||
err = amdgpu_amdkfd_gpuvm_sync_memory(dev->kgd, (struct kgd_mem *) mem, true);
|
||||
err = amdgpu_amdkfd_gpuvm_sync_memory(dev->adev, (struct kgd_mem *) mem, true);
|
||||
if (err) {
|
||||
pr_debug("Sync memory failed, wait interrupted by user signal\n");
|
||||
goto sync_memory_failed;
|
||||
@ -1593,7 +1593,7 @@ static int kfd_ioctl_unmap_memory_from_gpu(struct file *filep,
|
||||
goto get_mem_obj_from_handle_failed;
|
||||
}
|
||||
err = amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
|
||||
peer->kgd, (struct kgd_mem *)mem, peer_pdd->drm_priv);
|
||||
peer->adev, (struct kgd_mem *)mem, peer_pdd->drm_priv);
|
||||
if (err) {
|
||||
pr_err("Failed to unmap from gpu %d/%d\n",
|
||||
i, args->n_devices);
|
||||
@ -1603,8 +1603,8 @@ static int kfd_ioctl_unmap_memory_from_gpu(struct file *filep,
|
||||
}
|
||||
mutex_unlock(&p->mutex);
|
||||
|
||||
if (dev->device_info->asic_family == CHIP_ALDEBARAN) {
|
||||
err = amdgpu_amdkfd_gpuvm_sync_memory(dev->kgd,
|
||||
if (KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 2)) {
|
||||
err = amdgpu_amdkfd_gpuvm_sync_memory(dev->adev,
|
||||
(struct kgd_mem *) mem, true);
|
||||
if (err) {
|
||||
pr_debug("Sync memory failed, wait interrupted by user signal\n");
|
||||
@ -1680,7 +1680,7 @@ static int kfd_ioctl_get_dmabuf_info(struct file *filep,
|
||||
{
|
||||
struct kfd_ioctl_get_dmabuf_info_args *args = data;
|
||||
struct kfd_dev *dev = NULL;
|
||||
struct kgd_dev *dma_buf_kgd;
|
||||
struct amdgpu_device *dmabuf_adev;
|
||||
void *metadata_buffer = NULL;
|
||||
uint32_t flags;
|
||||
unsigned int i;
|
||||
@ -1700,15 +1700,15 @@ static int kfd_ioctl_get_dmabuf_info(struct file *filep,
|
||||
}
|
||||
|
||||
/* Get dmabuf info from KGD */
|
||||
r = amdgpu_amdkfd_get_dmabuf_info(dev->kgd, args->dmabuf_fd,
|
||||
&dma_buf_kgd, &args->size,
|
||||
r = amdgpu_amdkfd_get_dmabuf_info(dev->adev, args->dmabuf_fd,
|
||||
&dmabuf_adev, &args->size,
|
||||
metadata_buffer, args->metadata_size,
|
||||
&args->metadata_size, &flags);
|
||||
if (r)
|
||||
goto exit;
|
||||
|
||||
/* Reverse-lookup gpu_id from kgd pointer */
|
||||
dev = kfd_device_by_kgd(dma_buf_kgd);
|
||||
dev = kfd_device_by_adev(dmabuf_adev);
|
||||
if (!dev) {
|
||||
r = -EINVAL;
|
||||
goto exit;
|
||||
@ -1758,7 +1758,7 @@ static int kfd_ioctl_import_dmabuf(struct file *filep,
|
||||
goto err_unlock;
|
||||
}
|
||||
|
||||
r = amdgpu_amdkfd_gpuvm_import_dmabuf(dev->kgd, dmabuf,
|
||||
r = amdgpu_amdkfd_gpuvm_import_dmabuf(dev->adev, dmabuf,
|
||||
args->va_addr, pdd->drm_priv,
|
||||
(struct kgd_mem **)&mem, &size,
|
||||
NULL);
|
||||
@ -1779,7 +1779,7 @@ static int kfd_ioctl_import_dmabuf(struct file *filep,
|
||||
return 0;
|
||||
|
||||
err_free:
|
||||
amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->kgd, (struct kgd_mem *)mem,
|
||||
amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->adev, (struct kgd_mem *)mem,
|
||||
pdd->drm_priv, NULL);
|
||||
err_unlock:
|
||||
mutex_unlock(&p->mutex);
|
||||
@ -2066,7 +2066,7 @@ static int kfd_mmio_mmap(struct kfd_dev *dev, struct kfd_process *process,
|
||||
if (vma->vm_end - vma->vm_start != PAGE_SIZE)
|
||||
return -EINVAL;
|
||||
|
||||
address = amdgpu_amdkfd_get_mmio_remap_phys_addr(dev->kgd);
|
||||
address = dev->adev->rmmio_remap.bus_addr;
|
||||
|
||||
vma->vm_flags |= VM_IO | VM_DONTCOPY | VM_DONTEXPAND | VM_NORESERVE |
|
||||
VM_DONTDUMP | VM_PFNMAP;
|
||||
|
||||
@ -1340,7 +1340,7 @@ static int kfd_fill_gpu_cache_info(struct kfd_dev *kdev,
|
||||
int ret;
|
||||
unsigned int num_cu_shared;
|
||||
|
||||
switch (kdev->device_info->asic_family) {
|
||||
switch (kdev->adev->asic_type) {
|
||||
case CHIP_KAVERI:
|
||||
pcache_info = kaveri_cache_info;
|
||||
num_of_cache_types = ARRAY_SIZE(kaveri_cache_info);
|
||||
@ -1377,67 +1377,71 @@ static int kfd_fill_gpu_cache_info(struct kfd_dev *kdev,
|
||||
pcache_info = vegam_cache_info;
|
||||
num_of_cache_types = ARRAY_SIZE(vegam_cache_info);
|
||||
break;
|
||||
case CHIP_VEGA10:
|
||||
pcache_info = vega10_cache_info;
|
||||
num_of_cache_types = ARRAY_SIZE(vega10_cache_info);
|
||||
break;
|
||||
case CHIP_VEGA12:
|
||||
pcache_info = vega12_cache_info;
|
||||
num_of_cache_types = ARRAY_SIZE(vega12_cache_info);
|
||||
break;
|
||||
case CHIP_VEGA20:
|
||||
case CHIP_ARCTURUS:
|
||||
pcache_info = vega20_cache_info;
|
||||
num_of_cache_types = ARRAY_SIZE(vega20_cache_info);
|
||||
break;
|
||||
case CHIP_ALDEBARAN:
|
||||
pcache_info = aldebaran_cache_info;
|
||||
num_of_cache_types = ARRAY_SIZE(aldebaran_cache_info);
|
||||
break;
|
||||
case CHIP_RAVEN:
|
||||
pcache_info = raven_cache_info;
|
||||
num_of_cache_types = ARRAY_SIZE(raven_cache_info);
|
||||
break;
|
||||
case CHIP_RENOIR:
|
||||
pcache_info = renoir_cache_info;
|
||||
num_of_cache_types = ARRAY_SIZE(renoir_cache_info);
|
||||
break;
|
||||
case CHIP_NAVI10:
|
||||
case CHIP_NAVI12:
|
||||
case CHIP_CYAN_SKILLFISH:
|
||||
pcache_info = navi10_cache_info;
|
||||
num_of_cache_types = ARRAY_SIZE(navi10_cache_info);
|
||||
break;
|
||||
case CHIP_NAVI14:
|
||||
pcache_info = navi14_cache_info;
|
||||
num_of_cache_types = ARRAY_SIZE(navi14_cache_info);
|
||||
break;
|
||||
case CHIP_SIENNA_CICHLID:
|
||||
pcache_info = sienna_cichlid_cache_info;
|
||||
num_of_cache_types = ARRAY_SIZE(sienna_cichlid_cache_info);
|
||||
break;
|
||||
case CHIP_NAVY_FLOUNDER:
|
||||
pcache_info = navy_flounder_cache_info;
|
||||
num_of_cache_types = ARRAY_SIZE(navy_flounder_cache_info);
|
||||
break;
|
||||
case CHIP_DIMGREY_CAVEFISH:
|
||||
pcache_info = dimgrey_cavefish_cache_info;
|
||||
num_of_cache_types = ARRAY_SIZE(dimgrey_cavefish_cache_info);
|
||||
break;
|
||||
case CHIP_VANGOGH:
|
||||
pcache_info = vangogh_cache_info;
|
||||
num_of_cache_types = ARRAY_SIZE(vangogh_cache_info);
|
||||
break;
|
||||
case CHIP_BEIGE_GOBY:
|
||||
pcache_info = beige_goby_cache_info;
|
||||
num_of_cache_types = ARRAY_SIZE(beige_goby_cache_info);
|
||||
break;
|
||||
case CHIP_YELLOW_CARP:
|
||||
pcache_info = yellow_carp_cache_info;
|
||||
num_of_cache_types = ARRAY_SIZE(yellow_carp_cache_info);
|
||||
break;
|
||||
default:
|
||||
return -EINVAL;
|
||||
switch(KFD_GC_VERSION(kdev)) {
|
||||
case IP_VERSION(9, 0, 1):
|
||||
pcache_info = vega10_cache_info;
|
||||
num_of_cache_types = ARRAY_SIZE(vega10_cache_info);
|
||||
break;
|
||||
case IP_VERSION(9, 2, 1):
|
||||
pcache_info = vega12_cache_info;
|
||||
num_of_cache_types = ARRAY_SIZE(vega12_cache_info);
|
||||
break;
|
||||
case IP_VERSION(9, 4, 0):
|
||||
case IP_VERSION(9, 4, 1):
|
||||
pcache_info = vega20_cache_info;
|
||||
num_of_cache_types = ARRAY_SIZE(vega20_cache_info);
|
||||
break;
|
||||
case IP_VERSION(9, 4, 2):
|
||||
pcache_info = aldebaran_cache_info;
|
||||
num_of_cache_types = ARRAY_SIZE(aldebaran_cache_info);
|
||||
break;
|
||||
case IP_VERSION(9, 1, 0):
|
||||
case IP_VERSION(9, 2, 2):
|
||||
pcache_info = raven_cache_info;
|
||||
num_of_cache_types = ARRAY_SIZE(raven_cache_info);
|
||||
break;
|
||||
case IP_VERSION(9, 3, 0):
|
||||
pcache_info = renoir_cache_info;
|
||||
num_of_cache_types = ARRAY_SIZE(renoir_cache_info);
|
||||
break;
|
||||
case IP_VERSION(10, 1, 10):
|
||||
case IP_VERSION(10, 1, 2):
|
||||
case IP_VERSION(10, 1, 3):
|
||||
pcache_info = navi10_cache_info;
|
||||
num_of_cache_types = ARRAY_SIZE(navi10_cache_info);
|
||||
break;
|
||||
case IP_VERSION(10, 1, 1):
|
||||
pcache_info = navi14_cache_info;
|
||||
num_of_cache_types = ARRAY_SIZE(navi14_cache_info);
|
||||
break;
|
||||
case IP_VERSION(10, 3, 0):
|
||||
pcache_info = sienna_cichlid_cache_info;
|
||||
num_of_cache_types = ARRAY_SIZE(sienna_cichlid_cache_info);
|
||||
break;
|
||||
case IP_VERSION(10, 3, 2):
|
||||
pcache_info = navy_flounder_cache_info;
|
||||
num_of_cache_types = ARRAY_SIZE(navy_flounder_cache_info);
|
||||
break;
|
||||
case IP_VERSION(10, 3, 4):
|
||||
pcache_info = dimgrey_cavefish_cache_info;
|
||||
num_of_cache_types = ARRAY_SIZE(dimgrey_cavefish_cache_info);
|
||||
break;
|
||||
case IP_VERSION(10, 3, 1):
|
||||
pcache_info = vangogh_cache_info;
|
||||
num_of_cache_types = ARRAY_SIZE(vangogh_cache_info);
|
||||
break;
|
||||
case IP_VERSION(10, 3, 5):
|
||||
pcache_info = beige_goby_cache_info;
|
||||
num_of_cache_types = ARRAY_SIZE(beige_goby_cache_info);
|
||||
break;
|
||||
case IP_VERSION(10, 3, 3):
|
||||
pcache_info = yellow_carp_cache_info;
|
||||
num_of_cache_types = ARRAY_SIZE(yellow_carp_cache_info);
|
||||
break;
|
||||
default:
|
||||
return -EINVAL;
|
||||
}
|
||||
}
|
||||
|
||||
*size_filled = 0;
|
||||
@ -1963,8 +1967,6 @@ static int kfd_fill_gpu_direct_io_link_to_cpu(int *avail_size,
|
||||
struct crat_subtype_iolink *sub_type_hdr,
|
||||
uint32_t proximity_domain)
|
||||
{
|
||||
struct amdgpu_device *adev = (struct amdgpu_device *)kdev->kgd;
|
||||
|
||||
*avail_size -= sizeof(struct crat_subtype_iolink);
|
||||
if (*avail_size < 0)
|
||||
return -ENOMEM;
|
||||
@ -1981,7 +1983,7 @@ static int kfd_fill_gpu_direct_io_link_to_cpu(int *avail_size,
|
||||
/* Fill in IOLINK subtype.
|
||||
* TODO: Fill-in other fields of iolink subtype
|
||||
*/
|
||||
if (adev->gmc.xgmi.connected_to_cpu) {
|
||||
if (kdev->adev->gmc.xgmi.connected_to_cpu) {
|
||||
/*
|
||||
* with host gpu xgmi link, host can access gpu memory whether
|
||||
* or not pcie bar type is large, so always create bidirectional
|
||||
@ -1990,19 +1992,19 @@ static int kfd_fill_gpu_direct_io_link_to_cpu(int *avail_size,
|
||||
sub_type_hdr->flags |= CRAT_IOLINK_FLAGS_BI_DIRECTIONAL;
|
||||
sub_type_hdr->io_interface_type = CRAT_IOLINK_TYPE_XGMI;
|
||||
sub_type_hdr->num_hops_xgmi = 1;
|
||||
if (adev->asic_type == CHIP_ALDEBARAN) {
|
||||
if (KFD_GC_VERSION(kdev) == IP_VERSION(9, 4, 2)) {
|
||||
sub_type_hdr->minimum_bandwidth_mbs =
|
||||
amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(
|
||||
kdev->kgd, NULL, true);
|
||||
kdev->adev, NULL, true);
|
||||
sub_type_hdr->maximum_bandwidth_mbs =
|
||||
sub_type_hdr->minimum_bandwidth_mbs;
|
||||
}
|
||||
} else {
|
||||
sub_type_hdr->io_interface_type = CRAT_IOLINK_TYPE_PCIEXPRESS;
|
||||
sub_type_hdr->minimum_bandwidth_mbs =
|
||||
amdgpu_amdkfd_get_pcie_bandwidth_mbytes(kdev->kgd, true);
|
||||
amdgpu_amdkfd_get_pcie_bandwidth_mbytes(kdev->adev, true);
|
||||
sub_type_hdr->maximum_bandwidth_mbs =
|
||||
amdgpu_amdkfd_get_pcie_bandwidth_mbytes(kdev->kgd, false);
|
||||
amdgpu_amdkfd_get_pcie_bandwidth_mbytes(kdev->adev, false);
|
||||
}
|
||||
|
||||
sub_type_hdr->proximity_domain_from = proximity_domain;
|
||||
@ -2044,11 +2046,11 @@ static int kfd_fill_gpu_xgmi_link_to_gpu(int *avail_size,
|
||||
sub_type_hdr->proximity_domain_from = proximity_domain_from;
|
||||
sub_type_hdr->proximity_domain_to = proximity_domain_to;
|
||||
sub_type_hdr->num_hops_xgmi =
|
||||
amdgpu_amdkfd_get_xgmi_hops_count(kdev->kgd, peer_kdev->kgd);
|
||||
amdgpu_amdkfd_get_xgmi_hops_count(kdev->adev, peer_kdev->adev);
|
||||
sub_type_hdr->maximum_bandwidth_mbs =
|
||||
amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(kdev->kgd, peer_kdev->kgd, false);
|
||||
amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(kdev->adev, peer_kdev->adev, false);
|
||||
sub_type_hdr->minimum_bandwidth_mbs = sub_type_hdr->maximum_bandwidth_mbs ?
|
||||
amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(kdev->kgd, NULL, true) : 0;
|
||||
amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(kdev->adev, NULL, true) : 0;
|
||||
|
||||
return 0;
|
||||
}
|
||||
@ -2114,7 +2116,7 @@ static int kfd_create_vcrat_image_gpu(void *pcrat_image,
|
||||
cu->flags |= CRAT_CU_FLAGS_GPU_PRESENT;
|
||||
cu->proximity_domain = proximity_domain;
|
||||
|
||||
amdgpu_amdkfd_get_cu_info(kdev->kgd, &cu_info);
|
||||
amdgpu_amdkfd_get_cu_info(kdev->adev, &cu_info);
|
||||
cu->num_simd_per_cu = cu_info.simd_per_cu;
|
||||
cu->num_simd_cores = cu_info.simd_per_cu * cu_info.cu_active_number;
|
||||
cu->max_waves_simd = cu_info.max_waves_per_simd;
|
||||
@ -2145,7 +2147,7 @@ static int kfd_create_vcrat_image_gpu(void *pcrat_image,
|
||||
* report the total FB size (public+private) as a single
|
||||
* private heap.
|
||||
*/
|
||||
amdgpu_amdkfd_get_local_mem_info(kdev->kgd, &local_mem_info);
|
||||
amdgpu_amdkfd_get_local_mem_info(kdev->adev, &local_mem_info);
|
||||
sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
|
||||
sub_type_hdr->length);
|
||||
|
||||
|
||||
@ -41,7 +41,7 @@
|
||||
|
||||
static void dbgdev_address_watch_disable_nodiq(struct kfd_dev *dev)
|
||||
{
|
||||
dev->kfd2kgd->address_watch_disable(dev->kgd);
|
||||
dev->kfd2kgd->address_watch_disable(dev->adev);
|
||||
}
|
||||
|
||||
static int dbgdev_diq_submit_ib(struct kfd_dbgdev *dbgdev,
|
||||
@ -322,7 +322,7 @@ static int dbgdev_address_watch_nodiq(struct kfd_dbgdev *dbgdev,
|
||||
pr_debug("\t\t%30s\n", "* * * * * * * * * * * * * * * * * *");
|
||||
|
||||
pdd->dev->kfd2kgd->address_watch_execute(
|
||||
dbgdev->dev->kgd,
|
||||
dbgdev->dev->adev,
|
||||
i,
|
||||
cntl.u32All,
|
||||
addrHi.u32All,
|
||||
@ -420,7 +420,7 @@ static int dbgdev_address_watch_diq(struct kfd_dbgdev *dbgdev,
|
||||
|
||||
aw_reg_add_dword =
|
||||
dbgdev->dev->kfd2kgd->address_watch_get_offset(
|
||||
dbgdev->dev->kgd,
|
||||
dbgdev->dev->adev,
|
||||
i,
|
||||
ADDRESS_WATCH_REG_CNTL);
|
||||
|
||||
@ -431,7 +431,7 @@ static int dbgdev_address_watch_diq(struct kfd_dbgdev *dbgdev,
|
||||
|
||||
aw_reg_add_dword =
|
||||
dbgdev->dev->kfd2kgd->address_watch_get_offset(
|
||||
dbgdev->dev->kgd,
|
||||
dbgdev->dev->adev,
|
||||
i,
|
||||
ADDRESS_WATCH_REG_ADDR_HI);
|
||||
|
||||
@ -441,7 +441,7 @@ static int dbgdev_address_watch_diq(struct kfd_dbgdev *dbgdev,
|
||||
|
||||
aw_reg_add_dword =
|
||||
dbgdev->dev->kfd2kgd->address_watch_get_offset(
|
||||
dbgdev->dev->kgd,
|
||||
dbgdev->dev->adev,
|
||||
i,
|
||||
ADDRESS_WATCH_REG_ADDR_LO);
|
||||
|
||||
@ -457,7 +457,7 @@ static int dbgdev_address_watch_diq(struct kfd_dbgdev *dbgdev,
|
||||
|
||||
aw_reg_add_dword =
|
||||
dbgdev->dev->kfd2kgd->address_watch_get_offset(
|
||||
dbgdev->dev->kgd,
|
||||
dbgdev->dev->adev,
|
||||
i,
|
||||
ADDRESS_WATCH_REG_CNTL);
|
||||
|
||||
@ -752,7 +752,7 @@ static int dbgdev_wave_control_nodiq(struct kfd_dbgdev *dbgdev,
|
||||
|
||||
pr_debug("\t\t %30s\n", "* * * * * * * * * * * * * * * * * *");
|
||||
|
||||
return dbgdev->dev->kfd2kgd->wave_control_execute(dbgdev->dev->kgd,
|
||||
return dbgdev->dev->kfd2kgd->wave_control_execute(dbgdev->dev->adev,
|
||||
reg_gfx_index.u32All,
|
||||
reg_sq_cmd.u32All);
|
||||
}
|
||||
@ -784,7 +784,7 @@ int dbgdev_wave_reset_wavefronts(struct kfd_dev *dev, struct kfd_process *p)
|
||||
|
||||
for (vmid = first_vmid_to_scan; vmid <= last_vmid_to_scan; vmid++) {
|
||||
status = dev->kfd2kgd->get_atc_vmid_pasid_mapping_info
|
||||
(dev->kgd, vmid, &queried_pasid);
|
||||
(dev->adev, vmid, &queried_pasid);
|
||||
|
||||
if (status && queried_pasid == p->pasid) {
|
||||
pr_debug("Killing wave fronts of vmid %d and pasid 0x%x\n",
|
||||
@ -811,7 +811,7 @@ int dbgdev_wave_reset_wavefronts(struct kfd_dev *dev, struct kfd_process *p)
|
||||
/* for non DIQ we need to patch the VMID: */
|
||||
reg_sq_cmd.bits.vm_id = vmid;
|
||||
|
||||
dev->kfd2kgd->wave_control_execute(dev->kgd,
|
||||
dev->kfd2kgd->wave_control_execute(dev->adev,
|
||||
reg_gfx_index.u32All,
|
||||
reg_sq_cmd.u32All);
|
||||
|
||||
|
||||
@ -55,7 +55,6 @@ extern const struct kfd2kgd_calls gfx_v10_3_kfd2kgd;
|
||||
|
||||
#ifdef KFD_SUPPORT_IOMMU_V2
|
||||
static const struct kfd_device_info kaveri_device_info = {
|
||||
.asic_family = CHIP_KAVERI,
|
||||
.asic_name = "kaveri",
|
||||
.gfx_target_version = 70000,
|
||||
.max_pasid_bits = 16,
|
||||
@ -69,13 +68,10 @@ static const struct kfd_device_info kaveri_device_info = {
|
||||
.supports_cwsr = false,
|
||||
.needs_iommu_device = true,
|
||||
.needs_pci_atomics = false,
|
||||
.num_sdma_engines = 2,
|
||||
.num_xgmi_sdma_engines = 0,
|
||||
.num_sdma_queues_per_engine = 2,
|
||||
};
|
||||
|
||||
static const struct kfd_device_info carrizo_device_info = {
|
||||
.asic_family = CHIP_CARRIZO,
|
||||
.asic_name = "carrizo",
|
||||
.gfx_target_version = 80001,
|
||||
.max_pasid_bits = 16,
|
||||
@ -89,13 +85,10 @@ static const struct kfd_device_info carrizo_device_info = {
|
||||
.supports_cwsr = true,
|
||||
.needs_iommu_device = true,
|
||||
.needs_pci_atomics = false,
|
||||
.num_sdma_engines = 2,
|
||||
.num_xgmi_sdma_engines = 0,
|
||||
.num_sdma_queues_per_engine = 2,
|
||||
};
|
||||
|
||||
static const struct kfd_device_info raven_device_info = {
|
||||
.asic_family = CHIP_RAVEN,
|
||||
.asic_name = "raven",
|
||||
.gfx_target_version = 90002,
|
||||
.max_pasid_bits = 16,
|
||||
@ -108,15 +101,12 @@ static const struct kfd_device_info raven_device_info = {
|
||||
.supports_cwsr = true,
|
||||
.needs_iommu_device = true,
|
||||
.needs_pci_atomics = true,
|
||||
.num_sdma_engines = 1,
|
||||
.num_xgmi_sdma_engines = 0,
|
||||
.num_sdma_queues_per_engine = 2,
|
||||
};
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_DRM_AMDGPU_CIK
|
||||
static const struct kfd_device_info hawaii_device_info = {
|
||||
.asic_family = CHIP_HAWAII,
|
||||
.asic_name = "hawaii",
|
||||
.gfx_target_version = 70001,
|
||||
.max_pasid_bits = 16,
|
||||
@ -130,14 +120,11 @@ static const struct kfd_device_info hawaii_device_info = {
|
||||
.supports_cwsr = false,
|
||||
.needs_iommu_device = false,
|
||||
.needs_pci_atomics = false,
|
||||
.num_sdma_engines = 2,
|
||||
.num_xgmi_sdma_engines = 0,
|
||||
.num_sdma_queues_per_engine = 2,
|
||||
};
|
||||
#endif
|
||||
|
||||
static const struct kfd_device_info tonga_device_info = {
|
||||
.asic_family = CHIP_TONGA,
|
||||
.asic_name = "tonga",
|
||||
.gfx_target_version = 80002,
|
||||
.max_pasid_bits = 16,
|
||||
@ -150,13 +137,10 @@ static const struct kfd_device_info tonga_device_info = {
|
||||
.supports_cwsr = false,
|
||||
.needs_iommu_device = false,
|
||||
.needs_pci_atomics = true,
|
||||
.num_sdma_engines = 2,
|
||||
.num_xgmi_sdma_engines = 0,
|
||||
.num_sdma_queues_per_engine = 2,
|
||||
};
|
||||
|
||||
static const struct kfd_device_info fiji_device_info = {
|
||||
.asic_family = CHIP_FIJI,
|
||||
.asic_name = "fiji",
|
||||
.gfx_target_version = 80003,
|
||||
.max_pasid_bits = 16,
|
||||
@ -169,13 +153,10 @@ static const struct kfd_device_info fiji_device_info = {
|
||||
.supports_cwsr = true,
|
||||
.needs_iommu_device = false,
|
||||
.needs_pci_atomics = true,
|
||||
.num_sdma_engines = 2,
|
||||
.num_xgmi_sdma_engines = 0,
|
||||
.num_sdma_queues_per_engine = 2,
|
||||
};
|
||||
|
||||
static const struct kfd_device_info fiji_vf_device_info = {
|
||||
.asic_family = CHIP_FIJI,
|
||||
.asic_name = "fiji",
|
||||
.gfx_target_version = 80003,
|
||||
.max_pasid_bits = 16,
|
||||
@ -188,14 +169,11 @@ static const struct kfd_device_info fiji_vf_device_info = {
|
||||
.supports_cwsr = true,
|
||||
.needs_iommu_device = false,
|
||||
.needs_pci_atomics = false,
|
||||
.num_sdma_engines = 2,
|
||||
.num_xgmi_sdma_engines = 0,
|
||||
.num_sdma_queues_per_engine = 2,
|
||||
};
|
||||
|
||||
|
||||
static const struct kfd_device_info polaris10_device_info = {
|
||||
.asic_family = CHIP_POLARIS10,
|
||||
.asic_name = "polaris10",
|
||||
.gfx_target_version = 80003,
|
||||
.max_pasid_bits = 16,
|
||||
@ -208,13 +186,10 @@ static const struct kfd_device_info polaris10_device_info = {
|
||||
.supports_cwsr = true,
|
||||
.needs_iommu_device = false,
|
||||
.needs_pci_atomics = true,
|
||||
.num_sdma_engines = 2,
|
||||
.num_xgmi_sdma_engines = 0,
|
||||
.num_sdma_queues_per_engine = 2,
|
||||
};
|
||||
|
||||
static const struct kfd_device_info polaris10_vf_device_info = {
|
||||
.asic_family = CHIP_POLARIS10,
|
||||
.asic_name = "polaris10",
|
||||
.gfx_target_version = 80003,
|
||||
.max_pasid_bits = 16,
|
||||
@ -227,13 +202,10 @@ static const struct kfd_device_info polaris10_vf_device_info = {
|
||||
.supports_cwsr = true,
|
||||
.needs_iommu_device = false,
|
||||
.needs_pci_atomics = false,
|
||||
.num_sdma_engines = 2,
|
||||
.num_xgmi_sdma_engines = 0,
|
||||
.num_sdma_queues_per_engine = 2,
|
||||
};
|
||||
|
||||
static const struct kfd_device_info polaris11_device_info = {
|
||||
.asic_family = CHIP_POLARIS11,
|
||||
.asic_name = "polaris11",
|
||||
.gfx_target_version = 80003,
|
||||
.max_pasid_bits = 16,
|
||||
@ -246,13 +218,10 @@ static const struct kfd_device_info polaris11_device_info = {
|
||||
.supports_cwsr = true,
|
||||
.needs_iommu_device = false,
|
||||
.needs_pci_atomics = true,
|
||||
.num_sdma_engines = 2,
|
||||
.num_xgmi_sdma_engines = 0,
|
||||
.num_sdma_queues_per_engine = 2,
|
||||
};
|
||||
|
||||
static const struct kfd_device_info polaris12_device_info = {
|
||||
.asic_family = CHIP_POLARIS12,
|
||||
.asic_name = "polaris12",
|
||||
.gfx_target_version = 80003,
|
||||
.max_pasid_bits = 16,
|
||||
@ -265,13 +234,10 @@ static const struct kfd_device_info polaris12_device_info = {
|
||||
.supports_cwsr = true,
|
||||
.needs_iommu_device = false,
|
||||
.needs_pci_atomics = true,
|
||||
.num_sdma_engines = 2,
|
||||
.num_xgmi_sdma_engines = 0,
|
||||
.num_sdma_queues_per_engine = 2,
|
||||
};
|
||||
|
||||
static const struct kfd_device_info vegam_device_info = {
|
||||
.asic_family = CHIP_VEGAM,
|
||||
.asic_name = "vegam",
|
||||
.gfx_target_version = 80003,
|
||||
.max_pasid_bits = 16,
|
||||
@ -284,13 +250,10 @@ static const struct kfd_device_info vegam_device_info = {
|
||||
.supports_cwsr = true,
|
||||
.needs_iommu_device = false,
|
||||
.needs_pci_atomics = true,
|
||||
.num_sdma_engines = 2,
|
||||
.num_xgmi_sdma_engines = 0,
|
||||
.num_sdma_queues_per_engine = 2,
|
||||
};
|
||||
|
||||
static const struct kfd_device_info vega10_device_info = {
|
||||
.asic_family = CHIP_VEGA10,
|
||||
.asic_name = "vega10",
|
||||
.gfx_target_version = 90000,
|
||||
.max_pasid_bits = 16,
|
||||
@ -303,13 +266,10 @@ static const struct kfd_device_info vega10_device_info = {
|
||||
.supports_cwsr = true,
|
||||
.needs_iommu_device = false,
|
||||
.needs_pci_atomics = false,
|
||||
.num_sdma_engines = 2,
|
||||
.num_xgmi_sdma_engines = 0,
|
||||
.num_sdma_queues_per_engine = 2,
|
||||
};
|
||||
|
||||
static const struct kfd_device_info vega10_vf_device_info = {
|
||||
.asic_family = CHIP_VEGA10,
|
||||
.asic_name = "vega10",
|
||||
.gfx_target_version = 90000,
|
||||
.max_pasid_bits = 16,
|
||||
@ -322,13 +282,10 @@ static const struct kfd_device_info vega10_vf_device_info = {
|
||||
.supports_cwsr = true,
|
||||
.needs_iommu_device = false,
|
||||
.needs_pci_atomics = false,
|
||||
.num_sdma_engines = 2,
|
||||
.num_xgmi_sdma_engines = 0,
|
||||
.num_sdma_queues_per_engine = 2,
|
||||
};
|
||||
|
||||
static const struct kfd_device_info vega12_device_info = {
|
||||
.asic_family = CHIP_VEGA12,
|
||||
.asic_name = "vega12",
|
||||
.gfx_target_version = 90004,
|
||||
.max_pasid_bits = 16,
|
||||
@ -341,13 +298,10 @@ static const struct kfd_device_info vega12_device_info = {
|
||||
.supports_cwsr = true,
|
||||
.needs_iommu_device = false,
|
||||
.needs_pci_atomics = false,
|
||||
.num_sdma_engines = 2,
|
||||
.num_xgmi_sdma_engines = 0,
|
||||
.num_sdma_queues_per_engine = 2,
|
||||
};
|
||||
|
||||
static const struct kfd_device_info vega20_device_info = {
|
||||
.asic_family = CHIP_VEGA20,
|
||||
.asic_name = "vega20",
|
||||
.gfx_target_version = 90006,
|
||||
.max_pasid_bits = 16,
|
||||
@ -360,13 +314,10 @@ static const struct kfd_device_info vega20_device_info = {
|
||||
.supports_cwsr = true,
|
||||
.needs_iommu_device = false,
|
||||
.needs_pci_atomics = false,
|
||||
.num_sdma_engines = 2,
|
||||
.num_xgmi_sdma_engines = 0,
|
||||
.num_sdma_queues_per_engine = 8,
|
||||
};
|
||||
|
||||
static const struct kfd_device_info arcturus_device_info = {
|
||||
.asic_family = CHIP_ARCTURUS,
|
||||
.asic_name = "arcturus",
|
||||
.gfx_target_version = 90008,
|
||||
.max_pasid_bits = 16,
|
||||
@ -379,13 +330,10 @@ static const struct kfd_device_info arcturus_device_info = {
|
||||
.supports_cwsr = true,
|
||||
.needs_iommu_device = false,
|
||||
.needs_pci_atomics = false,
|
||||
.num_sdma_engines = 2,
|
||||
.num_xgmi_sdma_engines = 6,
|
||||
.num_sdma_queues_per_engine = 8,
|
||||
};
|
||||
|
||||
static const struct kfd_device_info aldebaran_device_info = {
|
||||
.asic_family = CHIP_ALDEBARAN,
|
||||
.asic_name = "aldebaran",
|
||||
.gfx_target_version = 90010,
|
||||
.max_pasid_bits = 16,
|
||||
@ -398,13 +346,10 @@ static const struct kfd_device_info aldebaran_device_info = {
|
||||
.supports_cwsr = true,
|
||||
.needs_iommu_device = false,
|
||||
.needs_pci_atomics = false,
|
||||
.num_sdma_engines = 2,
|
||||
.num_xgmi_sdma_engines = 3,
|
||||
.num_sdma_queues_per_engine = 8,
|
||||
};
|
||||
|
||||
static const struct kfd_device_info renoir_device_info = {
|
||||
.asic_family = CHIP_RENOIR,
|
||||
.asic_name = "renoir",
|
||||
.gfx_target_version = 90012,
|
||||
.max_pasid_bits = 16,
|
||||
@ -417,13 +362,10 @@ static const struct kfd_device_info renoir_device_info = {
|
||||
.supports_cwsr = true,
|
||||
.needs_iommu_device = false,
|
||||
.needs_pci_atomics = false,
|
||||
.num_sdma_engines = 1,
|
||||
.num_xgmi_sdma_engines = 0,
|
||||
.num_sdma_queues_per_engine = 2,
|
||||
};
|
||||
|
||||
static const struct kfd_device_info navi10_device_info = {
|
||||
.asic_family = CHIP_NAVI10,
|
||||
.asic_name = "navi10",
|
||||
.gfx_target_version = 100100,
|
||||
.max_pasid_bits = 16,
|
||||
@ -437,13 +379,10 @@ static const struct kfd_device_info navi10_device_info = {
|
||||
.supports_cwsr = true,
|
||||
.needs_pci_atomics = true,
|
||||
.no_atomic_fw_version = 145,
|
||||
.num_sdma_engines = 2,
|
||||
.num_xgmi_sdma_engines = 0,
|
||||
.num_sdma_queues_per_engine = 8,
|
||||
};
|
||||
|
||||
static const struct kfd_device_info navi12_device_info = {
|
||||
.asic_family = CHIP_NAVI12,
|
||||
.asic_name = "navi12",
|
||||
.gfx_target_version = 100101,
|
||||
.max_pasid_bits = 16,
|
||||
@ -457,13 +396,10 @@ static const struct kfd_device_info navi12_device_info = {
|
||||
.supports_cwsr = true,
|
||||
.needs_pci_atomics = true,
|
||||
.no_atomic_fw_version = 145,
|
||||
.num_sdma_engines = 2,
|
||||
.num_xgmi_sdma_engines = 0,
|
||||
.num_sdma_queues_per_engine = 8,
|
||||
};
|
||||
|
||||
static const struct kfd_device_info navi14_device_info = {
|
||||
.asic_family = CHIP_NAVI14,
|
||||
.asic_name = "navi14",
|
||||
.gfx_target_version = 100102,
|
||||
.max_pasid_bits = 16,
|
||||
@ -477,13 +413,10 @@ static const struct kfd_device_info navi14_device_info = {
|
||||
.supports_cwsr = true,
|
||||
.needs_pci_atomics = true,
|
||||
.no_atomic_fw_version = 145,
|
||||
.num_sdma_engines = 2,
|
||||
.num_xgmi_sdma_engines = 0,
|
||||
.num_sdma_queues_per_engine = 8,
|
||||
};
|
||||
|
||||
static const struct kfd_device_info sienna_cichlid_device_info = {
|
||||
.asic_family = CHIP_SIENNA_CICHLID,
|
||||
.asic_name = "sienna_cichlid",
|
||||
.gfx_target_version = 100300,
|
||||
.max_pasid_bits = 16,
|
||||
@ -497,13 +430,10 @@ static const struct kfd_device_info sienna_cichlid_device_info = {
|
||||
.supports_cwsr = true,
|
||||
.needs_pci_atomics = true,
|
||||
.no_atomic_fw_version = 92,
|
||||
.num_sdma_engines = 4,
|
||||
.num_xgmi_sdma_engines = 0,
|
||||
.num_sdma_queues_per_engine = 8,
|
||||
};
|
||||
|
||||
static const struct kfd_device_info navy_flounder_device_info = {
|
||||
.asic_family = CHIP_NAVY_FLOUNDER,
|
||||
.asic_name = "navy_flounder",
|
||||
.gfx_target_version = 100301,
|
||||
.max_pasid_bits = 16,
|
||||
@ -517,13 +447,10 @@ static const struct kfd_device_info navy_flounder_device_info = {
|
||||
.supports_cwsr = true,
|
||||
.needs_pci_atomics = true,
|
||||
.no_atomic_fw_version = 92,
|
||||
.num_sdma_engines = 2,
|
||||
.num_xgmi_sdma_engines = 0,
|
||||
.num_sdma_queues_per_engine = 8,
|
||||
};
|
||||
|
||||
static const struct kfd_device_info vangogh_device_info = {
|
||||
.asic_family = CHIP_VANGOGH,
|
||||
.asic_name = "vangogh",
|
||||
.gfx_target_version = 100303,
|
||||
.max_pasid_bits = 16,
|
||||
@ -537,13 +464,10 @@ static const struct kfd_device_info vangogh_device_info = {
|
||||
.supports_cwsr = true,
|
||||
.needs_pci_atomics = true,
|
||||
.no_atomic_fw_version = 92,
|
||||
.num_sdma_engines = 1,
|
||||
.num_xgmi_sdma_engines = 0,
|
||||
.num_sdma_queues_per_engine = 2,
|
||||
};
|
||||
|
||||
static const struct kfd_device_info dimgrey_cavefish_device_info = {
|
||||
.asic_family = CHIP_DIMGREY_CAVEFISH,
|
||||
.asic_name = "dimgrey_cavefish",
|
||||
.gfx_target_version = 100302,
|
||||
.max_pasid_bits = 16,
|
||||
@ -557,13 +481,10 @@ static const struct kfd_device_info dimgrey_cavefish_device_info = {
|
||||
.supports_cwsr = true,
|
||||
.needs_pci_atomics = true,
|
||||
.no_atomic_fw_version = 92,
|
||||
.num_sdma_engines = 2,
|
||||
.num_xgmi_sdma_engines = 0,
|
||||
.num_sdma_queues_per_engine = 8,
|
||||
};
|
||||
|
||||
static const struct kfd_device_info beige_goby_device_info = {
|
||||
.asic_family = CHIP_BEIGE_GOBY,
|
||||
.asic_name = "beige_goby",
|
||||
.gfx_target_version = 100304,
|
||||
.max_pasid_bits = 16,
|
||||
@ -577,13 +498,10 @@ static const struct kfd_device_info beige_goby_device_info = {
|
||||
.supports_cwsr = true,
|
||||
.needs_pci_atomics = true,
|
||||
.no_atomic_fw_version = 92,
|
||||
.num_sdma_engines = 1,
|
||||
.num_xgmi_sdma_engines = 0,
|
||||
.num_sdma_queues_per_engine = 8,
|
||||
};
|
||||
|
||||
static const struct kfd_device_info yellow_carp_device_info = {
|
||||
.asic_family = CHIP_YELLOW_CARP,
|
||||
.asic_name = "yellow_carp",
|
||||
.gfx_target_version = 100305,
|
||||
.max_pasid_bits = 16,
|
||||
@ -597,13 +515,10 @@ static const struct kfd_device_info yellow_carp_device_info = {
|
||||
.supports_cwsr = true,
|
||||
.needs_pci_atomics = true,
|
||||
.no_atomic_fw_version = 92,
|
||||
.num_sdma_engines = 1,
|
||||
.num_xgmi_sdma_engines = 0,
|
||||
.num_sdma_queues_per_engine = 2,
|
||||
};
|
||||
|
||||
static const struct kfd_device_info cyan_skillfish_device_info = {
|
||||
.asic_family = CHIP_CYAN_SKILLFISH,
|
||||
.asic_name = "cyan_skillfish",
|
||||
.gfx_target_version = 100103,
|
||||
.max_pasid_bits = 16,
|
||||
@ -616,8 +531,6 @@ static const struct kfd_device_info cyan_skillfish_device_info = {
|
||||
.needs_iommu_device = false,
|
||||
.supports_cwsr = true,
|
||||
.needs_pci_atomics = true,
|
||||
.num_sdma_engines = 2,
|
||||
.num_xgmi_sdma_engines = 0,
|
||||
.num_sdma_queues_per_engine = 8,
|
||||
};
|
||||
|
||||
@ -627,12 +540,11 @@ static void kfd_gtt_sa_fini(struct kfd_dev *kfd);
|
||||
|
||||
static int kfd_resume(struct kfd_dev *kfd);
|
||||
|
||||
struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd, bool vf)
|
||||
struct kfd_dev *kgd2kfd_probe(struct amdgpu_device *adev, bool vf)
|
||||
{
|
||||
struct kfd_dev *kfd;
|
||||
const struct kfd_device_info *device_info;
|
||||
const struct kfd2kgd_calls *f2g;
|
||||
struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
|
||||
struct pci_dev *pdev = adev->pdev;
|
||||
|
||||
switch (adev->asic_type) {
|
||||
@ -815,8 +727,12 @@ struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd, bool vf)
|
||||
}
|
||||
|
||||
if (!device_info || !f2g) {
|
||||
dev_err(kfd_device, "%s %s not supported in kfd\n",
|
||||
amdgpu_asic_name[adev->asic_type], vf ? "VF" : "");
|
||||
if (adev->ip_versions[GC_HWIP][0])
|
||||
dev_err(kfd_device, "GC IP %06x %s not supported in kfd\n",
|
||||
adev->ip_versions[GC_HWIP][0], vf ? "VF" : "");
|
||||
else
|
||||
dev_err(kfd_device, "%s %s not supported in kfd\n",
|
||||
amdgpu_asic_name[adev->asic_type], vf ? "VF" : "");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
@ -824,7 +740,7 @@ struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd, bool vf)
|
||||
if (!kfd)
|
||||
return NULL;
|
||||
|
||||
kfd->kgd = kgd;
|
||||
kfd->adev = adev;
|
||||
kfd->device_info = device_info;
|
||||
kfd->pdev = pdev;
|
||||
kfd->init_complete = false;
|
||||
@ -845,23 +761,23 @@ struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd, bool vf)
|
||||
static void kfd_cwsr_init(struct kfd_dev *kfd)
|
||||
{
|
||||
if (cwsr_enable && kfd->device_info->supports_cwsr) {
|
||||
if (kfd->device_info->asic_family < CHIP_VEGA10) {
|
||||
if (KFD_GC_VERSION(kfd) < IP_VERSION(9, 0, 1)) {
|
||||
BUILD_BUG_ON(sizeof(cwsr_trap_gfx8_hex) > PAGE_SIZE);
|
||||
kfd->cwsr_isa = cwsr_trap_gfx8_hex;
|
||||
kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx8_hex);
|
||||
} else if (kfd->device_info->asic_family == CHIP_ARCTURUS) {
|
||||
} else if (KFD_GC_VERSION(kfd) == IP_VERSION(9, 4, 1)) {
|
||||
BUILD_BUG_ON(sizeof(cwsr_trap_arcturus_hex) > PAGE_SIZE);
|
||||
kfd->cwsr_isa = cwsr_trap_arcturus_hex;
|
||||
kfd->cwsr_isa_size = sizeof(cwsr_trap_arcturus_hex);
|
||||
} else if (kfd->device_info->asic_family == CHIP_ALDEBARAN) {
|
||||
} else if (KFD_GC_VERSION(kfd) == IP_VERSION(9, 4, 2)) {
|
||||
BUILD_BUG_ON(sizeof(cwsr_trap_aldebaran_hex) > PAGE_SIZE);
|
||||
kfd->cwsr_isa = cwsr_trap_aldebaran_hex;
|
||||
kfd->cwsr_isa_size = sizeof(cwsr_trap_aldebaran_hex);
|
||||
} else if (kfd->device_info->asic_family < CHIP_NAVI10) {
|
||||
} else if (KFD_GC_VERSION(kfd) < IP_VERSION(10, 1, 1)) {
|
||||
BUILD_BUG_ON(sizeof(cwsr_trap_gfx9_hex) > PAGE_SIZE);
|
||||
kfd->cwsr_isa = cwsr_trap_gfx9_hex;
|
||||
kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx9_hex);
|
||||
} else if (kfd->device_info->asic_family < CHIP_SIENNA_CICHLID) {
|
||||
} else if (KFD_GC_VERSION(kfd) < IP_VERSION(10, 3, 0)) {
|
||||
BUILD_BUG_ON(sizeof(cwsr_trap_nv1x_hex) > PAGE_SIZE);
|
||||
kfd->cwsr_isa = cwsr_trap_nv1x_hex;
|
||||
kfd->cwsr_isa_size = sizeof(cwsr_trap_nv1x_hex);
|
||||
@ -882,18 +798,17 @@ static int kfd_gws_init(struct kfd_dev *kfd)
|
||||
if (kfd->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS)
|
||||
return 0;
|
||||
|
||||
if (hws_gws_support
|
||||
|| (kfd->device_info->asic_family == CHIP_VEGA10
|
||||
&& kfd->mec2_fw_version >= 0x81b3)
|
||||
|| (kfd->device_info->asic_family >= CHIP_VEGA12
|
||||
&& kfd->device_info->asic_family <= CHIP_RAVEN
|
||||
&& kfd->mec2_fw_version >= 0x1b3)
|
||||
|| (kfd->device_info->asic_family == CHIP_ARCTURUS
|
||||
&& kfd->mec2_fw_version >= 0x30)
|
||||
|| (kfd->device_info->asic_family == CHIP_ALDEBARAN
|
||||
&& kfd->mec2_fw_version >= 0x28))
|
||||
ret = amdgpu_amdkfd_alloc_gws(kfd->kgd,
|
||||
amdgpu_amdkfd_get_num_gws(kfd->kgd), &kfd->gws);
|
||||
if (hws_gws_support || (KFD_IS_SOC15(kfd) &&
|
||||
((KFD_GC_VERSION(kfd) == IP_VERSION(9, 0, 1)
|
||||
&& kfd->mec2_fw_version >= 0x81b3) ||
|
||||
(KFD_GC_VERSION(kfd) <= IP_VERSION(9, 4, 0)
|
||||
&& kfd->mec2_fw_version >= 0x1b3) ||
|
||||
(KFD_GC_VERSION(kfd) == IP_VERSION(9, 4, 1)
|
||||
&& kfd->mec2_fw_version >= 0x30) ||
|
||||
(KFD_GC_VERSION(kfd) == IP_VERSION(9, 4, 2)
|
||||
&& kfd->mec2_fw_version >= 0x28))))
|
||||
ret = amdgpu_amdkfd_alloc_gws(kfd->adev,
|
||||
kfd->adev->gds.gws_size, &kfd->gws);
|
||||
|
||||
return ret;
|
||||
}
|
||||
@ -910,11 +825,11 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
|
||||
unsigned int size, map_process_packet_size;
|
||||
|
||||
kfd->ddev = ddev;
|
||||
kfd->mec_fw_version = amdgpu_amdkfd_get_fw_version(kfd->kgd,
|
||||
kfd->mec_fw_version = amdgpu_amdkfd_get_fw_version(kfd->adev,
|
||||
KGD_ENGINE_MEC1);
|
||||
kfd->mec2_fw_version = amdgpu_amdkfd_get_fw_version(kfd->kgd,
|
||||
kfd->mec2_fw_version = amdgpu_amdkfd_get_fw_version(kfd->adev,
|
||||
KGD_ENGINE_MEC2);
|
||||
kfd->sdma_fw_version = amdgpu_amdkfd_get_fw_version(kfd->kgd,
|
||||
kfd->sdma_fw_version = amdgpu_amdkfd_get_fw_version(kfd->adev,
|
||||
KGD_ENGINE_SDMA1);
|
||||
kfd->shared_resources = *gpu_resources;
|
||||
|
||||
@ -927,7 +842,7 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
|
||||
* 32 and 64-bit requests are possible and must be
|
||||
* supported.
|
||||
*/
|
||||
kfd->pci_atomic_requested = amdgpu_amdkfd_have_atomics_support(kfd->kgd);
|
||||
kfd->pci_atomic_requested = amdgpu_amdkfd_have_atomics_support(kfd->adev);
|
||||
if (!kfd->pci_atomic_requested &&
|
||||
kfd->device_info->needs_pci_atomics &&
|
||||
(!kfd->device_info->no_atomic_fw_version ||
|
||||
@ -959,10 +874,9 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
|
||||
* calculate max size of runlist packet.
|
||||
* There can be only 2 packets at once
|
||||
*/
|
||||
map_process_packet_size =
|
||||
kfd->device_info->asic_family == CHIP_ALDEBARAN ?
|
||||
map_process_packet_size = KFD_GC_VERSION(kfd) == IP_VERSION(9, 4, 2) ?
|
||||
sizeof(struct pm4_mes_map_process_aldebaran) :
|
||||
sizeof(struct pm4_mes_map_process);
|
||||
sizeof(struct pm4_mes_map_process);
|
||||
size += (KFD_MAX_NUM_OF_PROCESSES * map_process_packet_size +
|
||||
max_num_of_queues_per_device * sizeof(struct pm4_mes_map_queues)
|
||||
+ sizeof(struct pm4_mes_runlist)) * 2;
|
||||
@ -974,7 +888,7 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
|
||||
size += 512 * 1024;
|
||||
|
||||
if (amdgpu_amdkfd_alloc_gtt_mem(
|
||||
kfd->kgd, size, &kfd->gtt_mem,
|
||||
kfd->adev, size, &kfd->gtt_mem,
|
||||
&kfd->gtt_start_gpu_addr, &kfd->gtt_start_cpu_ptr,
|
||||
false)) {
|
||||
dev_err(kfd_device, "Could not allocate %d bytes\n", size);
|
||||
@ -995,9 +909,9 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
|
||||
goto kfd_doorbell_error;
|
||||
}
|
||||
|
||||
kfd->hive_id = amdgpu_amdkfd_get_hive_id(kfd->kgd);
|
||||
kfd->hive_id = kfd->adev->gmc.xgmi.hive_id;
|
||||
|
||||
kfd->noretry = amdgpu_amdkfd_get_noretry(kfd->kgd);
|
||||
kfd->noretry = kfd->adev->gmc.noretry;
|
||||
|
||||
if (kfd_interrupt_init(kfd)) {
|
||||
dev_err(kfd_device, "Error initializing interrupts\n");
|
||||
@ -1015,7 +929,7 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
|
||||
*/
|
||||
if (kfd_gws_init(kfd)) {
|
||||
dev_err(kfd_device, "Could not allocate %d gws\n",
|
||||
amdgpu_amdkfd_get_num_gws(kfd->kgd));
|
||||
kfd->adev->gds.gws_size);
|
||||
goto gws_error;
|
||||
}
|
||||
|
||||
@ -1030,7 +944,7 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
|
||||
|
||||
kfd_cwsr_init(kfd);
|
||||
|
||||
svm_migrate_init((struct amdgpu_device *)kfd->kgd);
|
||||
svm_migrate_init(kfd->adev);
|
||||
|
||||
if(kgd2kfd_resume_iommu(kfd))
|
||||
goto device_iommu_error;
|
||||
@ -1068,10 +982,10 @@ kfd_interrupt_error:
|
||||
kfd_doorbell_error:
|
||||
kfd_gtt_sa_fini(kfd);
|
||||
kfd_gtt_sa_init_error:
|
||||
amdgpu_amdkfd_free_gtt_mem(kfd->kgd, kfd->gtt_mem);
|
||||
amdgpu_amdkfd_free_gtt_mem(kfd->adev, kfd->gtt_mem);
|
||||
alloc_gtt_mem_failure:
|
||||
if (kfd->gws)
|
||||
amdgpu_amdkfd_free_gws(kfd->kgd, kfd->gws);
|
||||
amdgpu_amdkfd_free_gws(kfd->adev, kfd->gws);
|
||||
dev_err(kfd_device,
|
||||
"device %x:%x NOT added due to errors\n",
|
||||
kfd->pdev->vendor, kfd->pdev->device);
|
||||
@ -1088,9 +1002,9 @@ void kgd2kfd_device_exit(struct kfd_dev *kfd)
|
||||
kfd_doorbell_fini(kfd);
|
||||
ida_destroy(&kfd->doorbell_ida);
|
||||
kfd_gtt_sa_fini(kfd);
|
||||
amdgpu_amdkfd_free_gtt_mem(kfd->kgd, kfd->gtt_mem);
|
||||
amdgpu_amdkfd_free_gtt_mem(kfd->adev, kfd->gtt_mem);
|
||||
if (kfd->gws)
|
||||
amdgpu_amdkfd_free_gws(kfd->kgd, kfd->gws);
|
||||
amdgpu_amdkfd_free_gws(kfd->adev, kfd->gws);
|
||||
}
|
||||
|
||||
kfree(kfd);
|
||||
@ -1526,7 +1440,7 @@ void kgd2kfd_set_sram_ecc_flag(struct kfd_dev *kfd)
|
||||
void kfd_inc_compute_active(struct kfd_dev *kfd)
|
||||
{
|
||||
if (atomic_inc_return(&kfd->compute_profile) == 1)
|
||||
amdgpu_amdkfd_set_compute_idle(kfd->kgd, false);
|
||||
amdgpu_amdkfd_set_compute_idle(kfd->adev, false);
|
||||
}
|
||||
|
||||
void kfd_dec_compute_active(struct kfd_dev *kfd)
|
||||
@ -1534,7 +1448,7 @@ void kfd_dec_compute_active(struct kfd_dev *kfd)
|
||||
int count = atomic_dec_return(&kfd->compute_profile);
|
||||
|
||||
if (count == 0)
|
||||
amdgpu_amdkfd_set_compute_idle(kfd->kgd, true);
|
||||
amdgpu_amdkfd_set_compute_idle(kfd->adev, true);
|
||||
WARN_ONCE(count < 0, "Compute profile ref. count error");
|
||||
}
|
||||
|
||||
@ -1544,6 +1458,26 @@ void kgd2kfd_smi_event_throttle(struct kfd_dev *kfd, uint64_t throttle_bitmask)
|
||||
kfd_smi_event_update_thermal_throttling(kfd, throttle_bitmask);
|
||||
}
|
||||
|
||||
/* kfd_get_num_sdma_engines returns the number of PCIe optimized SDMA and
|
||||
* kfd_get_num_xgmi_sdma_engines returns the number of XGMI SDMA.
|
||||
* When the device has more than two engines, we reserve two for PCIe to enable
|
||||
* full-duplex and the rest are used as XGMI.
|
||||
*/
|
||||
unsigned int kfd_get_num_sdma_engines(struct kfd_dev *kdev)
|
||||
{
|
||||
/* If XGMI is not supported, all SDMA engines are PCIe */
|
||||
if (!kdev->adev->gmc.xgmi.supported)
|
||||
return kdev->adev->sdma.num_instances;
|
||||
|
||||
return min(kdev->adev->sdma.num_instances, 2);
|
||||
}
|
||||
|
||||
unsigned int kfd_get_num_xgmi_sdma_engines(struct kfd_dev *kdev)
|
||||
{
|
||||
/* After reserved for PCIe, the rest of engines are XGMI */
|
||||
return kdev->adev->sdma.num_instances - kfd_get_num_sdma_engines(kdev);
|
||||
}
|
||||
|
||||
#if defined(CONFIG_DEBUG_FS)
|
||||
|
||||
/* This function will send a package to HIQ to hang the HWS
|
||||
|
||||
@ -99,38 +99,29 @@ unsigned int get_pipes_per_mec(struct device_queue_manager *dqm)
|
||||
return dqm->dev->shared_resources.num_pipe_per_mec;
|
||||
}
|
||||
|
||||
static unsigned int get_num_sdma_engines(struct device_queue_manager *dqm)
|
||||
{
|
||||
return dqm->dev->device_info->num_sdma_engines;
|
||||
}
|
||||
|
||||
static unsigned int get_num_xgmi_sdma_engines(struct device_queue_manager *dqm)
|
||||
{
|
||||
return dqm->dev->device_info->num_xgmi_sdma_engines;
|
||||
}
|
||||
|
||||
static unsigned int get_num_all_sdma_engines(struct device_queue_manager *dqm)
|
||||
{
|
||||
return get_num_sdma_engines(dqm) + get_num_xgmi_sdma_engines(dqm);
|
||||
return kfd_get_num_sdma_engines(dqm->dev) +
|
||||
kfd_get_num_xgmi_sdma_engines(dqm->dev);
|
||||
}
|
||||
|
||||
unsigned int get_num_sdma_queues(struct device_queue_manager *dqm)
|
||||
{
|
||||
return dqm->dev->device_info->num_sdma_engines
|
||||
* dqm->dev->device_info->num_sdma_queues_per_engine;
|
||||
return kfd_get_num_sdma_engines(dqm->dev) *
|
||||
dqm->dev->device_info->num_sdma_queues_per_engine;
|
||||
}
|
||||
|
||||
unsigned int get_num_xgmi_sdma_queues(struct device_queue_manager *dqm)
|
||||
{
|
||||
return dqm->dev->device_info->num_xgmi_sdma_engines
|
||||
* dqm->dev->device_info->num_sdma_queues_per_engine;
|
||||
return kfd_get_num_xgmi_sdma_engines(dqm->dev) *
|
||||
dqm->dev->device_info->num_sdma_queues_per_engine;
|
||||
}
|
||||
|
||||
void program_sh_mem_settings(struct device_queue_manager *dqm,
|
||||
struct qcm_process_device *qpd)
|
||||
{
|
||||
return dqm->dev->kfd2kgd->program_sh_mem_settings(
|
||||
dqm->dev->kgd, qpd->vmid,
|
||||
dqm->dev->adev, qpd->vmid,
|
||||
qpd->sh_mem_config,
|
||||
qpd->sh_mem_ape1_base,
|
||||
qpd->sh_mem_ape1_limit,
|
||||
@ -157,7 +148,7 @@ static int allocate_doorbell(struct qcm_process_device *qpd, struct queue *q)
|
||||
{
|
||||
struct kfd_dev *dev = qpd->dqm->dev;
|
||||
|
||||
if (!KFD_IS_SOC15(dev->device_info->asic_family)) {
|
||||
if (!KFD_IS_SOC15(dev)) {
|
||||
/* On pre-SOC15 chips we need to use the queue ID to
|
||||
* preserve the user mode ABI.
|
||||
*/
|
||||
@ -202,7 +193,7 @@ static void deallocate_doorbell(struct qcm_process_device *qpd,
|
||||
unsigned int old;
|
||||
struct kfd_dev *dev = qpd->dqm->dev;
|
||||
|
||||
if (!KFD_IS_SOC15(dev->device_info->asic_family) ||
|
||||
if (!KFD_IS_SOC15(dev) ||
|
||||
q->properties.type == KFD_QUEUE_TYPE_SDMA ||
|
||||
q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
|
||||
return;
|
||||
@ -216,7 +207,7 @@ static void program_trap_handler_settings(struct device_queue_manager *dqm,
|
||||
{
|
||||
if (dqm->dev->kfd2kgd->program_trap_handler_settings)
|
||||
dqm->dev->kfd2kgd->program_trap_handler_settings(
|
||||
dqm->dev->kgd, qpd->vmid,
|
||||
dqm->dev->adev, qpd->vmid,
|
||||
qpd->tba_addr, qpd->tma_addr);
|
||||
}
|
||||
|
||||
@ -250,21 +241,20 @@ static int allocate_vmid(struct device_queue_manager *dqm,
|
||||
|
||||
program_sh_mem_settings(dqm, qpd);
|
||||
|
||||
if (dqm->dev->device_info->asic_family >= CHIP_VEGA10 &&
|
||||
dqm->dev->cwsr_enabled)
|
||||
if (KFD_IS_SOC15(dqm->dev) && dqm->dev->cwsr_enabled)
|
||||
program_trap_handler_settings(dqm, qpd);
|
||||
|
||||
/* qpd->page_table_base is set earlier when register_process()
|
||||
* is called, i.e. when the first queue is created.
|
||||
*/
|
||||
dqm->dev->kfd2kgd->set_vm_context_page_table_base(dqm->dev->kgd,
|
||||
dqm->dev->kfd2kgd->set_vm_context_page_table_base(dqm->dev->adev,
|
||||
qpd->vmid,
|
||||
qpd->page_table_base);
|
||||
/* invalidate the VM context after pasid and vmid mapping is set up */
|
||||
kfd_flush_tlb(qpd_to_pdd(qpd), TLB_FLUSH_LEGACY);
|
||||
|
||||
if (dqm->dev->kfd2kgd->set_scratch_backing_va)
|
||||
dqm->dev->kfd2kgd->set_scratch_backing_va(dqm->dev->kgd,
|
||||
dqm->dev->kfd2kgd->set_scratch_backing_va(dqm->dev->adev,
|
||||
qpd->sh_hidden_private_base, qpd->vmid);
|
||||
|
||||
return 0;
|
||||
@ -283,7 +273,7 @@ static int flush_texture_cache_nocpsch(struct kfd_dev *kdev,
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
return amdgpu_amdkfd_submit_ib(kdev->kgd, KGD_ENGINE_MEC1, qpd->vmid,
|
||||
return amdgpu_amdkfd_submit_ib(kdev->adev, KGD_ENGINE_MEC1, qpd->vmid,
|
||||
qpd->ib_base, (uint32_t *)qpd->ib_kaddr,
|
||||
pmf->release_mem_size / sizeof(uint32_t));
|
||||
}
|
||||
@ -293,7 +283,7 @@ static void deallocate_vmid(struct device_queue_manager *dqm,
|
||||
struct queue *q)
|
||||
{
|
||||
/* On GFX v7, CP doesn't flush TC at dequeue */
|
||||
if (q->device->device_info->asic_family == CHIP_HAWAII)
|
||||
if (q->device->adev->asic_type == CHIP_HAWAII)
|
||||
if (flush_texture_cache_nocpsch(q->device, qpd))
|
||||
pr_err("Failed to flush TC\n");
|
||||
|
||||
@ -776,7 +766,7 @@ static int restore_process_queues_nocpsch(struct device_queue_manager *dqm,
|
||||
|
||||
if (!list_empty(&qpd->queues_list)) {
|
||||
dqm->dev->kfd2kgd->set_vm_context_page_table_base(
|
||||
dqm->dev->kgd,
|
||||
dqm->dev->adev,
|
||||
qpd->vmid,
|
||||
qpd->page_table_base);
|
||||
kfd_flush_tlb(pdd, TLB_FLUSH_LEGACY);
|
||||
@ -954,7 +944,7 @@ set_pasid_vmid_mapping(struct device_queue_manager *dqm, u32 pasid,
|
||||
unsigned int vmid)
|
||||
{
|
||||
return dqm->dev->kfd2kgd->set_pasid_vmid_mapping(
|
||||
dqm->dev->kgd, pasid, vmid);
|
||||
dqm->dev->adev, pasid, vmid);
|
||||
}
|
||||
|
||||
static void init_interrupts(struct device_queue_manager *dqm)
|
||||
@ -963,7 +953,7 @@ static void init_interrupts(struct device_queue_manager *dqm)
|
||||
|
||||
for (i = 0 ; i < get_pipes_per_mec(dqm) ; i++)
|
||||
if (is_pipe_enabled(dqm, 0, i))
|
||||
dqm->dev->kfd2kgd->init_interrupts(dqm->dev->kgd, i);
|
||||
dqm->dev->kfd2kgd->init_interrupts(dqm->dev->adev, i);
|
||||
}
|
||||
|
||||
static int initialize_nocpsch(struct device_queue_manager *dqm)
|
||||
@ -1017,7 +1007,7 @@ static int start_nocpsch(struct device_queue_manager *dqm)
|
||||
pr_info("SW scheduler is used");
|
||||
init_interrupts(dqm);
|
||||
|
||||
if (dqm->dev->device_info->asic_family == CHIP_HAWAII)
|
||||
if (dqm->dev->adev->asic_type == CHIP_HAWAII)
|
||||
return pm_init(&dqm->packet_mgr, dqm);
|
||||
dqm->sched_running = true;
|
||||
|
||||
@ -1026,7 +1016,7 @@ static int start_nocpsch(struct device_queue_manager *dqm)
|
||||
|
||||
static int stop_nocpsch(struct device_queue_manager *dqm)
|
||||
{
|
||||
if (dqm->dev->device_info->asic_family == CHIP_HAWAII)
|
||||
if (dqm->dev->adev->asic_type == CHIP_HAWAII)
|
||||
pm_uninit(&dqm->packet_mgr, false);
|
||||
dqm->sched_running = false;
|
||||
|
||||
@ -1055,9 +1045,9 @@ static int allocate_sdma_queue(struct device_queue_manager *dqm,
|
||||
dqm->sdma_bitmap &= ~(1ULL << bit);
|
||||
q->sdma_id = bit;
|
||||
q->properties.sdma_engine_id = q->sdma_id %
|
||||
get_num_sdma_engines(dqm);
|
||||
kfd_get_num_sdma_engines(dqm->dev);
|
||||
q->properties.sdma_queue_id = q->sdma_id /
|
||||
get_num_sdma_engines(dqm);
|
||||
kfd_get_num_sdma_engines(dqm->dev);
|
||||
} else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
|
||||
if (dqm->xgmi_sdma_bitmap == 0) {
|
||||
pr_err("No more XGMI SDMA queue to allocate\n");
|
||||
@ -1072,10 +1062,11 @@ static int allocate_sdma_queue(struct device_queue_manager *dqm,
|
||||
* assumes the first N engines are always
|
||||
* PCIe-optimized ones
|
||||
*/
|
||||
q->properties.sdma_engine_id = get_num_sdma_engines(dqm) +
|
||||
q->sdma_id % get_num_xgmi_sdma_engines(dqm);
|
||||
q->properties.sdma_engine_id =
|
||||
kfd_get_num_sdma_engines(dqm->dev) +
|
||||
q->sdma_id % kfd_get_num_xgmi_sdma_engines(dqm->dev);
|
||||
q->properties.sdma_queue_id = q->sdma_id /
|
||||
get_num_xgmi_sdma_engines(dqm);
|
||||
kfd_get_num_xgmi_sdma_engines(dqm->dev);
|
||||
}
|
||||
|
||||
pr_debug("SDMA engine id: %d\n", q->properties.sdma_engine_id);
|
||||
@ -1132,7 +1123,7 @@ static int set_sched_resources(struct device_queue_manager *dqm)
|
||||
|
||||
res.queue_mask |= 1ull
|
||||
<< amdgpu_queue_mask_bit_to_set_resource_bit(
|
||||
(struct amdgpu_device *)dqm->dev->kgd, i);
|
||||
dqm->dev->adev, i);
|
||||
}
|
||||
res.gws_mask = ~0ull;
|
||||
res.oac_mask = res.gds_heap_base = res.gds_heap_size = 0;
|
||||
@ -1850,7 +1841,7 @@ static int allocate_hiq_sdma_mqd(struct device_queue_manager *dqm)
|
||||
dev->device_info->num_sdma_queues_per_engine +
|
||||
dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ]->mqd_size;
|
||||
|
||||
retval = amdgpu_amdkfd_alloc_gtt_mem(dev->kgd, size,
|
||||
retval = amdgpu_amdkfd_alloc_gtt_mem(dev->adev, size,
|
||||
&(mem_obj->gtt_mem), &(mem_obj->gpu_addr),
|
||||
(void *)&(mem_obj->cpu_ptr), false);
|
||||
|
||||
@ -1867,7 +1858,7 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
|
||||
if (!dqm)
|
||||
return NULL;
|
||||
|
||||
switch (dev->device_info->asic_family) {
|
||||
switch (dev->adev->asic_type) {
|
||||
/* HWS is not available on Hawaii. */
|
||||
case CHIP_HAWAII:
|
||||
/* HWS depends on CWSR for timely dequeue. CWSR is not
|
||||
@ -1930,7 +1921,7 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
|
||||
goto out_free;
|
||||
}
|
||||
|
||||
switch (dev->device_info->asic_family) {
|
||||
switch (dev->adev->asic_type) {
|
||||
case CHIP_CARRIZO:
|
||||
device_queue_manager_init_vi(&dqm->asic_ops);
|
||||
break;
|
||||
@ -1952,31 +1943,16 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
|
||||
device_queue_manager_init_vi_tonga(&dqm->asic_ops);
|
||||
break;
|
||||
|
||||
case CHIP_VEGA10:
|
||||
case CHIP_VEGA12:
|
||||
case CHIP_VEGA20:
|
||||
case CHIP_RAVEN:
|
||||
case CHIP_RENOIR:
|
||||
case CHIP_ARCTURUS:
|
||||
case CHIP_ALDEBARAN:
|
||||
device_queue_manager_init_v9(&dqm->asic_ops);
|
||||
break;
|
||||
case CHIP_NAVI10:
|
||||
case CHIP_NAVI12:
|
||||
case CHIP_NAVI14:
|
||||
case CHIP_SIENNA_CICHLID:
|
||||
case CHIP_NAVY_FLOUNDER:
|
||||
case CHIP_VANGOGH:
|
||||
case CHIP_DIMGREY_CAVEFISH:
|
||||
case CHIP_BEIGE_GOBY:
|
||||
case CHIP_YELLOW_CARP:
|
||||
case CHIP_CYAN_SKILLFISH:
|
||||
device_queue_manager_init_v10_navi10(&dqm->asic_ops);
|
||||
break;
|
||||
default:
|
||||
WARN(1, "Unexpected ASIC family %u",
|
||||
dev->device_info->asic_family);
|
||||
goto out_free;
|
||||
if (KFD_GC_VERSION(dev) >= IP_VERSION(10, 1, 1))
|
||||
device_queue_manager_init_v10_navi10(&dqm->asic_ops);
|
||||
else if (KFD_GC_VERSION(dev) >= IP_VERSION(9, 0, 1))
|
||||
device_queue_manager_init_v9(&dqm->asic_ops);
|
||||
else {
|
||||
WARN(1, "Unexpected ASIC family %u",
|
||||
dev->adev->asic_type);
|
||||
goto out_free;
|
||||
}
|
||||
}
|
||||
|
||||
if (init_mqd_managers(dqm))
|
||||
@ -2000,7 +1976,7 @@ static void deallocate_hiq_sdma_mqd(struct kfd_dev *dev,
|
||||
{
|
||||
WARN(!mqd, "No hiq sdma mqd trunk to free");
|
||||
|
||||
amdgpu_amdkfd_free_gtt_mem(dev->kgd, mqd->gtt_mem);
|
||||
amdgpu_amdkfd_free_gtt_mem(dev->adev, mqd->gtt_mem);
|
||||
}
|
||||
|
||||
void device_queue_manager_uninit(struct device_queue_manager *dqm)
|
||||
@ -2031,7 +2007,7 @@ static void kfd_process_hw_exception(struct work_struct *work)
|
||||
{
|
||||
struct device_queue_manager *dqm = container_of(work,
|
||||
struct device_queue_manager, hw_exception_work);
|
||||
amdgpu_amdkfd_gpu_reset(dqm->dev->kgd);
|
||||
amdgpu_amdkfd_gpu_reset(dqm->dev->adev);
|
||||
}
|
||||
|
||||
#if defined(CONFIG_DEBUG_FS)
|
||||
@ -2070,7 +2046,7 @@ int dqm_debugfs_hqds(struct seq_file *m, void *data)
|
||||
return 0;
|
||||
}
|
||||
|
||||
r = dqm->dev->kfd2kgd->hqd_dump(dqm->dev->kgd,
|
||||
r = dqm->dev->kfd2kgd->hqd_dump(dqm->dev->adev,
|
||||
KFD_CIK_HIQ_PIPE, KFD_CIK_HIQ_QUEUE,
|
||||
&dump, &n_regs);
|
||||
if (!r) {
|
||||
@ -2092,7 +2068,7 @@ int dqm_debugfs_hqds(struct seq_file *m, void *data)
|
||||
continue;
|
||||
|
||||
r = dqm->dev->kfd2kgd->hqd_dump(
|
||||
dqm->dev->kgd, pipe, queue, &dump, &n_regs);
|
||||
dqm->dev->adev, pipe, queue, &dump, &n_regs);
|
||||
if (r)
|
||||
break;
|
||||
|
||||
@ -2109,7 +2085,7 @@ int dqm_debugfs_hqds(struct seq_file *m, void *data)
|
||||
queue < dqm->dev->device_info->num_sdma_queues_per_engine;
|
||||
queue++) {
|
||||
r = dqm->dev->kfd2kgd->hqd_sdma_dump(
|
||||
dqm->dev->kgd, pipe, queue, &dump, &n_regs);
|
||||
dqm->dev->adev, pipe, queue, &dump, &n_regs);
|
||||
if (r)
|
||||
break;
|
||||
|
||||
|
||||
@ -62,7 +62,7 @@ static int update_qpd_v9(struct device_queue_manager *dqm,
|
||||
SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
|
||||
SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT;
|
||||
|
||||
if (dqm->dev->device_info->asic_family == CHIP_ALDEBARAN) {
|
||||
if (KFD_GC_VERSION(dqm->dev) == IP_VERSION(9, 4, 2)) {
|
||||
/* Aldebaran can safely support different XNACK modes
|
||||
* per process
|
||||
*/
|
||||
|
||||
@ -935,8 +935,10 @@ void kfd_signal_iommu_event(struct kfd_dev *dev, u32 pasid,
|
||||
/* Workaround on Raven to not kill the process when memory is freed
|
||||
* before IOMMU is able to finish processing all the excessive PPRs
|
||||
*/
|
||||
if (dev->device_info->asic_family != CHIP_RAVEN &&
|
||||
dev->device_info->asic_family != CHIP_RENOIR) {
|
||||
|
||||
if (KFD_GC_VERSION(dev) != IP_VERSION(9, 1, 0) &&
|
||||
KFD_GC_VERSION(dev) != IP_VERSION(9, 2, 2) &&
|
||||
KFD_GC_VERSION(dev) != IP_VERSION(9, 3, 0)) {
|
||||
mutex_lock(&p->event_mutex);
|
||||
|
||||
/* Lookup events by type and signal them */
|
||||
|
||||
@ -394,7 +394,7 @@ int kfd_init_apertures(struct kfd_process *process)
|
||||
pdd->gpuvm_base = pdd->gpuvm_limit = 0;
|
||||
pdd->scratch_base = pdd->scratch_limit = 0;
|
||||
} else {
|
||||
switch (dev->device_info->asic_family) {
|
||||
switch (dev->adev->asic_type) {
|
||||
case CHIP_KAVERI:
|
||||
case CHIP_HAWAII:
|
||||
case CHIP_CARRIZO:
|
||||
@ -406,29 +406,14 @@ int kfd_init_apertures(struct kfd_process *process)
|
||||
case CHIP_VEGAM:
|
||||
kfd_init_apertures_vi(pdd, id);
|
||||
break;
|
||||
case CHIP_VEGA10:
|
||||
case CHIP_VEGA12:
|
||||
case CHIP_VEGA20:
|
||||
case CHIP_RAVEN:
|
||||
case CHIP_RENOIR:
|
||||
case CHIP_ARCTURUS:
|
||||
case CHIP_ALDEBARAN:
|
||||
case CHIP_NAVI10:
|
||||
case CHIP_NAVI12:
|
||||
case CHIP_NAVI14:
|
||||
case CHIP_SIENNA_CICHLID:
|
||||
case CHIP_NAVY_FLOUNDER:
|
||||
case CHIP_VANGOGH:
|
||||
case CHIP_DIMGREY_CAVEFISH:
|
||||
case CHIP_BEIGE_GOBY:
|
||||
case CHIP_YELLOW_CARP:
|
||||
case CHIP_CYAN_SKILLFISH:
|
||||
kfd_init_apertures_v9(pdd, id);
|
||||
break;
|
||||
default:
|
||||
WARN(1, "Unexpected ASIC family %u",
|
||||
dev->device_info->asic_family);
|
||||
return -EINVAL;
|
||||
if (KFD_GC_VERSION(dev) >= IP_VERSION(9, 0, 1))
|
||||
kfd_init_apertures_v9(pdd, id);
|
||||
else {
|
||||
WARN(1, "Unexpected ASIC family %u",
|
||||
dev->adev->asic_type);
|
||||
return -EINVAL;
|
||||
}
|
||||
}
|
||||
|
||||
if (!dev->use_iommu_v2) {
|
||||
|
||||
@ -231,7 +231,7 @@ static void event_interrupt_wq_v9(struct kfd_dev *dev,
|
||||
if (sq_intr_err != SQ_INTERRUPT_ERROR_TYPE_ILLEGAL_INST &&
|
||||
sq_intr_err != SQ_INTERRUPT_ERROR_TYPE_MEMVIOL) {
|
||||
kfd_signal_poison_consumed_event(dev, pasid);
|
||||
amdgpu_amdkfd_ras_poison_consumption_handler(dev->kgd);
|
||||
amdgpu_amdkfd_ras_poison_consumption_handler(dev->adev);
|
||||
return;
|
||||
}
|
||||
break;
|
||||
@ -253,7 +253,7 @@ static void event_interrupt_wq_v9(struct kfd_dev *dev,
|
||||
kfd_signal_event_interrupt(pasid, context_id0 & 0xfffffff, 28);
|
||||
} else if (source_id == SOC15_INTSRC_SDMA_ECC) {
|
||||
kfd_signal_poison_consumed_event(dev, pasid);
|
||||
amdgpu_amdkfd_ras_poison_consumption_handler(dev->kgd);
|
||||
amdgpu_amdkfd_ras_poison_consumption_handler(dev->adev);
|
||||
return;
|
||||
}
|
||||
} else if (client_id == SOC15_IH_CLIENTID_VMC ||
|
||||
|
||||
@ -91,7 +91,7 @@ static bool kq_initialize(struct kernel_queue *kq, struct kfd_dev *dev,
|
||||
kq->pq_gpu_addr = kq->pq->gpu_addr;
|
||||
|
||||
/* For CIK family asics, kq->eop_mem is not needed */
|
||||
if (dev->device_info->asic_family > CHIP_MULLINS) {
|
||||
if (dev->adev->asic_type > CHIP_MULLINS) {
|
||||
retval = kfd_gtt_sa_allocate(dev, PAGE_SIZE, &kq->eop_mem);
|
||||
if (retval != 0)
|
||||
goto err_eop_allocate_vidmem;
|
||||
|
||||
@ -938,7 +938,7 @@ int svm_migrate_init(struct amdgpu_device *adev)
|
||||
void *r;
|
||||
|
||||
/* Page migration works on Vega10 or newer */
|
||||
if (kfddev->device_info->asic_family < CHIP_VEGA10)
|
||||
if (!KFD_IS_SOC15(kfddev))
|
||||
return -EINVAL;
|
||||
|
||||
pgmap = &kfddev->pgmap;
|
||||
|
||||
@ -100,7 +100,7 @@ void mqd_symmetrically_map_cu_mask(struct mqd_manager *mm,
|
||||
struct kfd_cu_info cu_info;
|
||||
uint32_t cu_per_sh[KFD_MAX_NUM_SE][KFD_MAX_NUM_SH_PER_SE] = {0};
|
||||
int i, se, sh, cu;
|
||||
amdgpu_amdkfd_get_cu_info(mm->dev->kgd, &cu_info);
|
||||
amdgpu_amdkfd_get_cu_info(mm->dev->adev, &cu_info);
|
||||
|
||||
if (cu_mask_count > cu_info.cu_active_number)
|
||||
cu_mask_count = cu_info.cu_active_number;
|
||||
|
||||
@ -171,7 +171,7 @@ static int load_mqd(struct mqd_manager *mm, void *mqd, uint32_t pipe_id,
|
||||
uint32_t wptr_shift = (p->format == KFD_QUEUE_FORMAT_AQL ? 4 : 0);
|
||||
uint32_t wptr_mask = (uint32_t)((p->queue_size / 4) - 1);
|
||||
|
||||
return mm->dev->kfd2kgd->hqd_load(mm->dev->kgd, mqd, pipe_id, queue_id,
|
||||
return mm->dev->kfd2kgd->hqd_load(mm->dev->adev, mqd, pipe_id, queue_id,
|
||||
(uint32_t __user *)p->write_ptr,
|
||||
wptr_shift, wptr_mask, mms);
|
||||
}
|
||||
@ -180,7 +180,7 @@ static int load_mqd_sdma(struct mqd_manager *mm, void *mqd,
|
||||
uint32_t pipe_id, uint32_t queue_id,
|
||||
struct queue_properties *p, struct mm_struct *mms)
|
||||
{
|
||||
return mm->dev->kfd2kgd->hqd_sdma_load(mm->dev->kgd, mqd,
|
||||
return mm->dev->kfd2kgd->hqd_sdma_load(mm->dev->adev, mqd,
|
||||
(uint32_t __user *)p->write_ptr,
|
||||
mms);
|
||||
}
|
||||
@ -276,7 +276,7 @@ static int destroy_mqd(struct mqd_manager *mm, void *mqd,
|
||||
unsigned int timeout, uint32_t pipe_id,
|
||||
uint32_t queue_id)
|
||||
{
|
||||
return mm->dev->kfd2kgd->hqd_destroy(mm->dev->kgd, mqd, type, timeout,
|
||||
return mm->dev->kfd2kgd->hqd_destroy(mm->dev->adev, mqd, type, timeout,
|
||||
pipe_id, queue_id);
|
||||
}
|
||||
|
||||
@ -289,7 +289,7 @@ static int destroy_mqd_sdma(struct mqd_manager *mm, void *mqd,
|
||||
unsigned int timeout, uint32_t pipe_id,
|
||||
uint32_t queue_id)
|
||||
{
|
||||
return mm->dev->kfd2kgd->hqd_sdma_destroy(mm->dev->kgd, mqd, timeout);
|
||||
return mm->dev->kfd2kgd->hqd_sdma_destroy(mm->dev->adev, mqd, timeout);
|
||||
}
|
||||
|
||||
static bool is_occupied(struct mqd_manager *mm, void *mqd,
|
||||
@ -297,7 +297,7 @@ static bool is_occupied(struct mqd_manager *mm, void *mqd,
|
||||
uint32_t queue_id)
|
||||
{
|
||||
|
||||
return mm->dev->kfd2kgd->hqd_is_occupied(mm->dev->kgd, queue_address,
|
||||
return mm->dev->kfd2kgd->hqd_is_occupied(mm->dev->adev, queue_address,
|
||||
pipe_id, queue_id);
|
||||
|
||||
}
|
||||
@ -306,7 +306,7 @@ static bool is_occupied_sdma(struct mqd_manager *mm, void *mqd,
|
||||
uint64_t queue_address, uint32_t pipe_id,
|
||||
uint32_t queue_id)
|
||||
{
|
||||
return mm->dev->kfd2kgd->hqd_sdma_is_occupied(mm->dev->kgd, mqd);
|
||||
return mm->dev->kfd2kgd->hqd_sdma_is_occupied(mm->dev->adev, mqd);
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
@ -148,7 +148,7 @@ static int load_mqd(struct mqd_manager *mm, void *mqd,
|
||||
/* AQL write pointer counts in 64B packets, PM4/CP counts in dwords. */
|
||||
uint32_t wptr_shift = (p->format == KFD_QUEUE_FORMAT_AQL ? 4 : 0);
|
||||
|
||||
r = mm->dev->kfd2kgd->hqd_load(mm->dev->kgd, mqd, pipe_id, queue_id,
|
||||
r = mm->dev->kfd2kgd->hqd_load(mm->dev->adev, mqd, pipe_id, queue_id,
|
||||
(uint32_t __user *)p->write_ptr,
|
||||
wptr_shift, 0, mms);
|
||||
return r;
|
||||
@ -158,7 +158,7 @@ static int hiq_load_mqd_kiq(struct mqd_manager *mm, void *mqd,
|
||||
uint32_t pipe_id, uint32_t queue_id,
|
||||
struct queue_properties *p, struct mm_struct *mms)
|
||||
{
|
||||
return mm->dev->kfd2kgd->hiq_mqd_load(mm->dev->kgd, mqd, pipe_id,
|
||||
return mm->dev->kfd2kgd->hiq_mqd_load(mm->dev->adev, mqd, pipe_id,
|
||||
queue_id, p->doorbell_off);
|
||||
}
|
||||
|
||||
@ -239,7 +239,7 @@ static int destroy_mqd(struct mqd_manager *mm, void *mqd,
|
||||
uint32_t queue_id)
|
||||
{
|
||||
return mm->dev->kfd2kgd->hqd_destroy
|
||||
(mm->dev->kgd, mqd, type, timeout,
|
||||
(mm->dev->adev, mqd, type, timeout,
|
||||
pipe_id, queue_id);
|
||||
}
|
||||
|
||||
@ -254,7 +254,7 @@ static bool is_occupied(struct mqd_manager *mm, void *mqd,
|
||||
uint32_t queue_id)
|
||||
{
|
||||
return mm->dev->kfd2kgd->hqd_is_occupied(
|
||||
mm->dev->kgd, queue_address,
|
||||
mm->dev->adev, queue_address,
|
||||
pipe_id, queue_id);
|
||||
}
|
||||
|
||||
@ -320,7 +320,7 @@ static int load_mqd_sdma(struct mqd_manager *mm, void *mqd,
|
||||
uint32_t pipe_id, uint32_t queue_id,
|
||||
struct queue_properties *p, struct mm_struct *mms)
|
||||
{
|
||||
return mm->dev->kfd2kgd->hqd_sdma_load(mm->dev->kgd, mqd,
|
||||
return mm->dev->kfd2kgd->hqd_sdma_load(mm->dev->adev, mqd,
|
||||
(uint32_t __user *)p->write_ptr,
|
||||
mms);
|
||||
}
|
||||
@ -363,14 +363,14 @@ static int destroy_mqd_sdma(struct mqd_manager *mm, void *mqd,
|
||||
unsigned int timeout, uint32_t pipe_id,
|
||||
uint32_t queue_id)
|
||||
{
|
||||
return mm->dev->kfd2kgd->hqd_sdma_destroy(mm->dev->kgd, mqd, timeout);
|
||||
return mm->dev->kfd2kgd->hqd_sdma_destroy(mm->dev->adev, mqd, timeout);
|
||||
}
|
||||
|
||||
static bool is_occupied_sdma(struct mqd_manager *mm, void *mqd,
|
||||
uint64_t queue_address, uint32_t pipe_id,
|
||||
uint32_t queue_id)
|
||||
{
|
||||
return mm->dev->kfd2kgd->hqd_sdma_is_occupied(mm->dev->kgd, mqd);
|
||||
return mm->dev->kfd2kgd->hqd_sdma_is_occupied(mm->dev->adev, mqd);
|
||||
}
|
||||
|
||||
#if defined(CONFIG_DEBUG_FS)
|
||||
|
||||
@ -108,7 +108,7 @@ static struct kfd_mem_obj *allocate_mqd(struct kfd_dev *kfd,
|
||||
mqd_mem_obj = kzalloc(sizeof(struct kfd_mem_obj), GFP_KERNEL);
|
||||
if (!mqd_mem_obj)
|
||||
return NULL;
|
||||
retval = amdgpu_amdkfd_alloc_gtt_mem(kfd->kgd,
|
||||
retval = amdgpu_amdkfd_alloc_gtt_mem(kfd->adev,
|
||||
ALIGN(q->ctl_stack_size, PAGE_SIZE) +
|
||||
ALIGN(sizeof(struct v9_mqd), PAGE_SIZE),
|
||||
&(mqd_mem_obj->gtt_mem),
|
||||
@ -199,7 +199,7 @@ static int load_mqd(struct mqd_manager *mm, void *mqd,
|
||||
/* AQL write pointer counts in 64B packets, PM4/CP counts in dwords. */
|
||||
uint32_t wptr_shift = (p->format == KFD_QUEUE_FORMAT_AQL ? 4 : 0);
|
||||
|
||||
return mm->dev->kfd2kgd->hqd_load(mm->dev->kgd, mqd, pipe_id, queue_id,
|
||||
return mm->dev->kfd2kgd->hqd_load(mm->dev->adev, mqd, pipe_id, queue_id,
|
||||
(uint32_t __user *)p->write_ptr,
|
||||
wptr_shift, 0, mms);
|
||||
}
|
||||
@ -208,7 +208,7 @@ static int hiq_load_mqd_kiq(struct mqd_manager *mm, void *mqd,
|
||||
uint32_t pipe_id, uint32_t queue_id,
|
||||
struct queue_properties *p, struct mm_struct *mms)
|
||||
{
|
||||
return mm->dev->kfd2kgd->hiq_mqd_load(mm->dev->kgd, mqd, pipe_id,
|
||||
return mm->dev->kfd2kgd->hiq_mqd_load(mm->dev->adev, mqd, pipe_id,
|
||||
queue_id, p->doorbell_off);
|
||||
}
|
||||
|
||||
@ -291,7 +291,7 @@ static int destroy_mqd(struct mqd_manager *mm, void *mqd,
|
||||
uint32_t queue_id)
|
||||
{
|
||||
return mm->dev->kfd2kgd->hqd_destroy
|
||||
(mm->dev->kgd, mqd, type, timeout,
|
||||
(mm->dev->adev, mqd, type, timeout,
|
||||
pipe_id, queue_id);
|
||||
}
|
||||
|
||||
@ -301,7 +301,7 @@ static void free_mqd(struct mqd_manager *mm, void *mqd,
|
||||
struct kfd_dev *kfd = mm->dev;
|
||||
|
||||
if (mqd_mem_obj->gtt_mem) {
|
||||
amdgpu_amdkfd_free_gtt_mem(kfd->kgd, mqd_mem_obj->gtt_mem);
|
||||
amdgpu_amdkfd_free_gtt_mem(kfd->adev, mqd_mem_obj->gtt_mem);
|
||||
kfree(mqd_mem_obj);
|
||||
} else {
|
||||
kfd_gtt_sa_free(mm->dev, mqd_mem_obj);
|
||||
@ -313,7 +313,7 @@ static bool is_occupied(struct mqd_manager *mm, void *mqd,
|
||||
uint32_t queue_id)
|
||||
{
|
||||
return mm->dev->kfd2kgd->hqd_is_occupied(
|
||||
mm->dev->kgd, queue_address,
|
||||
mm->dev->adev, queue_address,
|
||||
pipe_id, queue_id);
|
||||
}
|
||||
|
||||
@ -375,7 +375,7 @@ static int load_mqd_sdma(struct mqd_manager *mm, void *mqd,
|
||||
uint32_t pipe_id, uint32_t queue_id,
|
||||
struct queue_properties *p, struct mm_struct *mms)
|
||||
{
|
||||
return mm->dev->kfd2kgd->hqd_sdma_load(mm->dev->kgd, mqd,
|
||||
return mm->dev->kfd2kgd->hqd_sdma_load(mm->dev->adev, mqd,
|
||||
(uint32_t __user *)p->write_ptr,
|
||||
mms);
|
||||
}
|
||||
@ -418,14 +418,14 @@ static int destroy_mqd_sdma(struct mqd_manager *mm, void *mqd,
|
||||
unsigned int timeout, uint32_t pipe_id,
|
||||
uint32_t queue_id)
|
||||
{
|
||||
return mm->dev->kfd2kgd->hqd_sdma_destroy(mm->dev->kgd, mqd, timeout);
|
||||
return mm->dev->kfd2kgd->hqd_sdma_destroy(mm->dev->adev, mqd, timeout);
|
||||
}
|
||||
|
||||
static bool is_occupied_sdma(struct mqd_manager *mm, void *mqd,
|
||||
uint64_t queue_address, uint32_t pipe_id,
|
||||
uint32_t queue_id)
|
||||
{
|
||||
return mm->dev->kfd2kgd->hqd_sdma_is_occupied(mm->dev->kgd, mqd);
|
||||
return mm->dev->kfd2kgd->hqd_sdma_is_occupied(mm->dev->adev, mqd);
|
||||
}
|
||||
|
||||
#if defined(CONFIG_DEBUG_FS)
|
||||
|
||||
@ -162,7 +162,7 @@ static int load_mqd(struct mqd_manager *mm, void *mqd,
|
||||
uint32_t wptr_shift = (p->format == KFD_QUEUE_FORMAT_AQL ? 4 : 0);
|
||||
uint32_t wptr_mask = (uint32_t)((p->queue_size / 4) - 1);
|
||||
|
||||
return mm->dev->kfd2kgd->hqd_load(mm->dev->kgd, mqd, pipe_id, queue_id,
|
||||
return mm->dev->kfd2kgd->hqd_load(mm->dev->adev, mqd, pipe_id, queue_id,
|
||||
(uint32_t __user *)p->write_ptr,
|
||||
wptr_shift, wptr_mask, mms);
|
||||
}
|
||||
@ -265,7 +265,7 @@ static int destroy_mqd(struct mqd_manager *mm, void *mqd,
|
||||
uint32_t queue_id)
|
||||
{
|
||||
return mm->dev->kfd2kgd->hqd_destroy
|
||||
(mm->dev->kgd, mqd, type, timeout,
|
||||
(mm->dev->adev, mqd, type, timeout,
|
||||
pipe_id, queue_id);
|
||||
}
|
||||
|
||||
@ -280,7 +280,7 @@ static bool is_occupied(struct mqd_manager *mm, void *mqd,
|
||||
uint32_t queue_id)
|
||||
{
|
||||
return mm->dev->kfd2kgd->hqd_is_occupied(
|
||||
mm->dev->kgd, queue_address,
|
||||
mm->dev->adev, queue_address,
|
||||
pipe_id, queue_id);
|
||||
}
|
||||
|
||||
@ -347,7 +347,7 @@ static int load_mqd_sdma(struct mqd_manager *mm, void *mqd,
|
||||
uint32_t pipe_id, uint32_t queue_id,
|
||||
struct queue_properties *p, struct mm_struct *mms)
|
||||
{
|
||||
return mm->dev->kfd2kgd->hqd_sdma_load(mm->dev->kgd, mqd,
|
||||
return mm->dev->kfd2kgd->hqd_sdma_load(mm->dev->adev, mqd,
|
||||
(uint32_t __user *)p->write_ptr,
|
||||
mms);
|
||||
}
|
||||
@ -389,14 +389,14 @@ static int destroy_mqd_sdma(struct mqd_manager *mm, void *mqd,
|
||||
unsigned int timeout, uint32_t pipe_id,
|
||||
uint32_t queue_id)
|
||||
{
|
||||
return mm->dev->kfd2kgd->hqd_sdma_destroy(mm->dev->kgd, mqd, timeout);
|
||||
return mm->dev->kfd2kgd->hqd_sdma_destroy(mm->dev->adev, mqd, timeout);
|
||||
}
|
||||
|
||||
static bool is_occupied_sdma(struct mqd_manager *mm, void *mqd,
|
||||
uint64_t queue_address, uint32_t pipe_id,
|
||||
uint32_t queue_id)
|
||||
{
|
||||
return mm->dev->kfd2kgd->hqd_sdma_is_occupied(mm->dev->kgd, mqd);
|
||||
return mm->dev->kfd2kgd->hqd_sdma_is_occupied(mm->dev->adev, mqd);
|
||||
}
|
||||
|
||||
#if defined(CONFIG_DEBUG_FS)
|
||||
|
||||
@ -223,7 +223,7 @@ static int pm_create_runlist_ib(struct packet_manager *pm,
|
||||
|
||||
int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm)
|
||||
{
|
||||
switch (dqm->dev->device_info->asic_family) {
|
||||
switch (dqm->dev->adev->asic_type) {
|
||||
case CHIP_KAVERI:
|
||||
case CHIP_HAWAII:
|
||||
/* PM4 packet structures on CIK are the same as on VI */
|
||||
@ -236,31 +236,16 @@ int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm)
|
||||
case CHIP_VEGAM:
|
||||
pm->pmf = &kfd_vi_pm_funcs;
|
||||
break;
|
||||
case CHIP_VEGA10:
|
||||
case CHIP_VEGA12:
|
||||
case CHIP_VEGA20:
|
||||
case CHIP_RAVEN:
|
||||
case CHIP_RENOIR:
|
||||
case CHIP_ARCTURUS:
|
||||
case CHIP_NAVI10:
|
||||
case CHIP_NAVI12:
|
||||
case CHIP_NAVI14:
|
||||
case CHIP_SIENNA_CICHLID:
|
||||
case CHIP_NAVY_FLOUNDER:
|
||||
case CHIP_VANGOGH:
|
||||
case CHIP_DIMGREY_CAVEFISH:
|
||||
case CHIP_BEIGE_GOBY:
|
||||
case CHIP_YELLOW_CARP:
|
||||
case CHIP_CYAN_SKILLFISH:
|
||||
pm->pmf = &kfd_v9_pm_funcs;
|
||||
break;
|
||||
case CHIP_ALDEBARAN:
|
||||
pm->pmf = &kfd_aldebaran_pm_funcs;
|
||||
break;
|
||||
default:
|
||||
WARN(1, "Unexpected ASIC family %u",
|
||||
dqm->dev->device_info->asic_family);
|
||||
return -EINVAL;
|
||||
if (KFD_GC_VERSION(dqm->dev) == IP_VERSION(9, 4, 2))
|
||||
pm->pmf = &kfd_aldebaran_pm_funcs;
|
||||
else if (KFD_GC_VERSION(dqm->dev) >= IP_VERSION(9, 0, 1))
|
||||
pm->pmf = &kfd_v9_pm_funcs;
|
||||
else {
|
||||
WARN(1, "Unexpected ASIC family %u",
|
||||
dqm->dev->adev->asic_type);
|
||||
return -EINVAL;
|
||||
}
|
||||
}
|
||||
|
||||
pm->dqm = dqm;
|
||||
|
||||
@ -183,7 +183,8 @@ enum cache_policy {
|
||||
cache_policy_noncoherent
|
||||
};
|
||||
|
||||
#define KFD_IS_SOC15(chip) ((chip) >= CHIP_VEGA10)
|
||||
#define KFD_GC_VERSION(dev) ((dev)->adev->ip_versions[GC_HWIP][0])
|
||||
#define KFD_IS_SOC15(dev) ((KFD_GC_VERSION(dev)) >= (IP_VERSION(9, 0, 1)))
|
||||
|
||||
struct kfd_event_interrupt_class {
|
||||
bool (*interrupt_isr)(struct kfd_dev *dev,
|
||||
@ -194,7 +195,6 @@ struct kfd_event_interrupt_class {
|
||||
};
|
||||
|
||||
struct kfd_device_info {
|
||||
enum amd_asic_type asic_family;
|
||||
const char *asic_name;
|
||||
uint32_t gfx_target_version;
|
||||
const struct kfd_event_interrupt_class *event_interrupt_class;
|
||||
@ -208,11 +208,12 @@ struct kfd_device_info {
|
||||
bool needs_iommu_device;
|
||||
bool needs_pci_atomics;
|
||||
uint32_t no_atomic_fw_version;
|
||||
unsigned int num_sdma_engines;
|
||||
unsigned int num_xgmi_sdma_engines;
|
||||
unsigned int num_sdma_queues_per_engine;
|
||||
};
|
||||
|
||||
unsigned int kfd_get_num_sdma_engines(struct kfd_dev *kdev);
|
||||
unsigned int kfd_get_num_xgmi_sdma_engines(struct kfd_dev *kdev);
|
||||
|
||||
struct kfd_mem_obj {
|
||||
uint32_t range_start;
|
||||
uint32_t range_end;
|
||||
@ -228,7 +229,7 @@ struct kfd_vmid_info {
|
||||
};
|
||||
|
||||
struct kfd_dev {
|
||||
struct kgd_dev *kgd;
|
||||
struct amdgpu_device *adev;
|
||||
|
||||
const struct kfd_device_info *device_info;
|
||||
struct pci_dev *pdev;
|
||||
@ -766,7 +767,7 @@ struct svm_range_list {
|
||||
struct list_head deferred_range_list;
|
||||
spinlock_t deferred_list_lock;
|
||||
atomic_t evicted_ranges;
|
||||
bool drain_pagefaults;
|
||||
atomic_t drain_pagefaults;
|
||||
struct delayed_work restore_work;
|
||||
DECLARE_BITMAP(bitmap_supported, MAX_GPU_INSTANCE);
|
||||
struct task_struct *faulting_task;
|
||||
@ -891,7 +892,7 @@ struct kfd_process *kfd_lookup_process_by_pasid(u32 pasid);
|
||||
struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm);
|
||||
|
||||
int kfd_process_gpuidx_from_gpuid(struct kfd_process *p, uint32_t gpu_id);
|
||||
int kfd_process_gpuid_from_kgd(struct kfd_process *p,
|
||||
int kfd_process_gpuid_from_adev(struct kfd_process *p,
|
||||
struct amdgpu_device *adev, uint32_t *gpuid,
|
||||
uint32_t *gpuidx);
|
||||
static inline int kfd_process_gpuid_from_gpuidx(struct kfd_process *p,
|
||||
@ -984,7 +985,7 @@ struct kfd_topology_device *kfd_topology_device_by_proximity_domain(
|
||||
struct kfd_topology_device *kfd_topology_device_by_id(uint32_t gpu_id);
|
||||
struct kfd_dev *kfd_device_by_id(uint32_t gpu_id);
|
||||
struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev);
|
||||
struct kfd_dev *kfd_device_by_kgd(const struct kgd_dev *kgd);
|
||||
struct kfd_dev *kfd_device_by_adev(const struct amdgpu_device *adev);
|
||||
int kfd_topology_enum_kfd_devices(uint8_t idx, struct kfd_dev **kdev);
|
||||
int kfd_numa_node_to_apic_id(int numa_node_id);
|
||||
void kfd_double_confirm_iommu_support(struct kfd_dev *gpu);
|
||||
|
||||
@ -288,7 +288,7 @@ static int kfd_get_cu_occupancy(struct attribute *attr, char *buffer)
|
||||
/* Collect wave count from device if it supports */
|
||||
wave_cnt = 0;
|
||||
max_waves_per_cu = 0;
|
||||
dev->kfd2kgd->get_cu_occupancy(dev->kgd, proc->pasid, &wave_cnt,
|
||||
dev->kfd2kgd->get_cu_occupancy(dev->adev, proc->pasid, &wave_cnt,
|
||||
&max_waves_per_cu);
|
||||
|
||||
/* Translate wave count to number of compute units */
|
||||
@ -692,12 +692,12 @@ static void kfd_process_free_gpuvm(struct kgd_mem *mem,
|
||||
struct kfd_dev *dev = pdd->dev;
|
||||
|
||||
if (kptr) {
|
||||
amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(dev->kgd, mem);
|
||||
amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(dev->adev, mem);
|
||||
kptr = NULL;
|
||||
}
|
||||
|
||||
amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(dev->kgd, mem, pdd->drm_priv);
|
||||
amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->kgd, mem, pdd->drm_priv,
|
||||
amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(dev->adev, mem, pdd->drm_priv);
|
||||
amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->adev, mem, pdd->drm_priv,
|
||||
NULL);
|
||||
}
|
||||
|
||||
@ -714,24 +714,24 @@ static int kfd_process_alloc_gpuvm(struct kfd_process_device *pdd,
|
||||
struct kfd_dev *kdev = pdd->dev;
|
||||
int err;
|
||||
|
||||
err = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(kdev->kgd, gpu_va, size,
|
||||
err = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(kdev->adev, gpu_va, size,
|
||||
pdd->drm_priv, mem, NULL, flags);
|
||||
if (err)
|
||||
goto err_alloc_mem;
|
||||
|
||||
err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(kdev->kgd, *mem,
|
||||
err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(kdev->adev, *mem,
|
||||
pdd->drm_priv, NULL);
|
||||
if (err)
|
||||
goto err_map_mem;
|
||||
|
||||
err = amdgpu_amdkfd_gpuvm_sync_memory(kdev->kgd, *mem, true);
|
||||
err = amdgpu_amdkfd_gpuvm_sync_memory(kdev->adev, *mem, true);
|
||||
if (err) {
|
||||
pr_debug("Sync memory failed, wait interrupted by user signal\n");
|
||||
goto sync_memory_failed;
|
||||
}
|
||||
|
||||
if (kptr) {
|
||||
err = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(kdev->kgd,
|
||||
err = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(kdev->adev,
|
||||
(struct kgd_mem *)*mem, kptr, NULL);
|
||||
if (err) {
|
||||
pr_debug("Map GTT BO to kernel failed\n");
|
||||
@ -742,10 +742,10 @@ static int kfd_process_alloc_gpuvm(struct kfd_process_device *pdd,
|
||||
return err;
|
||||
|
||||
sync_memory_failed:
|
||||
amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(kdev->kgd, *mem, pdd->drm_priv);
|
||||
amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(kdev->adev, *mem, pdd->drm_priv);
|
||||
|
||||
err_map_mem:
|
||||
amdgpu_amdkfd_gpuvm_free_memory_of_gpu(kdev->kgd, *mem, pdd->drm_priv,
|
||||
amdgpu_amdkfd_gpuvm_free_memory_of_gpu(kdev->adev, *mem, pdd->drm_priv,
|
||||
NULL);
|
||||
err_alloc_mem:
|
||||
*mem = NULL;
|
||||
@ -940,10 +940,10 @@ static void kfd_process_device_free_bos(struct kfd_process_device *pdd)
|
||||
if (!peer_pdd->drm_priv)
|
||||
continue;
|
||||
amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
|
||||
peer_pdd->dev->kgd, mem, peer_pdd->drm_priv);
|
||||
peer_pdd->dev->adev, mem, peer_pdd->drm_priv);
|
||||
}
|
||||
|
||||
amdgpu_amdkfd_gpuvm_free_memory_of_gpu(pdd->dev->kgd, mem,
|
||||
amdgpu_amdkfd_gpuvm_free_memory_of_gpu(pdd->dev->adev, mem,
|
||||
pdd->drm_priv, NULL);
|
||||
kfd_process_device_remove_obj_handle(pdd, id);
|
||||
}
|
||||
@ -974,7 +974,7 @@ static void kfd_process_kunmap_signal_bo(struct kfd_process *p)
|
||||
if (!mem)
|
||||
goto out;
|
||||
|
||||
amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(kdev->kgd, mem);
|
||||
amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(kdev->adev, mem);
|
||||
|
||||
out:
|
||||
mutex_unlock(&p->mutex);
|
||||
@ -1003,7 +1003,7 @@ static void kfd_process_destroy_pdds(struct kfd_process *p)
|
||||
|
||||
if (pdd->drm_file) {
|
||||
amdgpu_amdkfd_gpuvm_release_process_vm(
|
||||
pdd->dev->kgd, pdd->drm_priv);
|
||||
pdd->dev->adev, pdd->drm_priv);
|
||||
fput(pdd->drm_file);
|
||||
}
|
||||
|
||||
@ -1317,14 +1317,13 @@ bool kfd_process_xnack_mode(struct kfd_process *p, bool supported)
|
||||
* support the SVM APIs and don't need to be considered
|
||||
* for the XNACK mode selection.
|
||||
*/
|
||||
if (dev->device_info->asic_family < CHIP_VEGA10)
|
||||
if (!KFD_IS_SOC15(dev))
|
||||
continue;
|
||||
/* Aldebaran can always support XNACK because it can support
|
||||
* per-process XNACK mode selection. But let the dev->noretry
|
||||
* setting still influence the default XNACK mode.
|
||||
*/
|
||||
if (supported &&
|
||||
dev->device_info->asic_family == CHIP_ALDEBARAN)
|
||||
if (supported && KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 2))
|
||||
continue;
|
||||
|
||||
/* GFXv10 and later GPUs do not support shader preemption
|
||||
@ -1332,7 +1331,7 @@ bool kfd_process_xnack_mode(struct kfd_process *p, bool supported)
|
||||
* management and memory-manager-related preemptions or
|
||||
* even deadlocks.
|
||||
*/
|
||||
if (dev->device_info->asic_family >= CHIP_NAVI10)
|
||||
if (KFD_GC_VERSION(dev) >= IP_VERSION(10, 1, 1))
|
||||
return false;
|
||||
|
||||
if (dev->noretry)
|
||||
@ -1431,7 +1430,7 @@ static int init_doorbell_bitmap(struct qcm_process_device *qpd,
|
||||
int range_start = dev->shared_resources.non_cp_doorbells_start;
|
||||
int range_end = dev->shared_resources.non_cp_doorbells_end;
|
||||
|
||||
if (!KFD_IS_SOC15(dev->device_info->asic_family))
|
||||
if (!KFD_IS_SOC15(dev))
|
||||
return 0;
|
||||
|
||||
qpd->doorbell_bitmap =
|
||||
@ -1547,7 +1546,7 @@ int kfd_process_device_init_vm(struct kfd_process_device *pdd,
|
||||
dev = pdd->dev;
|
||||
|
||||
ret = amdgpu_amdkfd_gpuvm_acquire_process_vm(
|
||||
dev->kgd, drm_file, p->pasid,
|
||||
dev->adev, drm_file, p->pasid,
|
||||
&p->kgd_process_info, &p->ef);
|
||||
if (ret) {
|
||||
pr_err("Failed to create process VM object\n");
|
||||
@ -1779,14 +1778,13 @@ int kfd_process_gpuidx_from_gpuid(struct kfd_process *p, uint32_t gpu_id)
|
||||
}
|
||||
|
||||
int
|
||||
kfd_process_gpuid_from_kgd(struct kfd_process *p, struct amdgpu_device *adev,
|
||||
kfd_process_gpuid_from_adev(struct kfd_process *p, struct amdgpu_device *adev,
|
||||
uint32_t *gpuid, uint32_t *gpuidx)
|
||||
{
|
||||
struct kgd_dev *kgd = (struct kgd_dev *)adev;
|
||||
int i;
|
||||
|
||||
for (i = 0; i < p->n_pdds; i++)
|
||||
if (p->pdds[i] && p->pdds[i]->dev->kgd == kgd) {
|
||||
if (p->pdds[i] && p->pdds[i]->dev->adev == adev) {
|
||||
*gpuid = p->pdds[i]->dev->id;
|
||||
*gpuidx = i;
|
||||
return 0;
|
||||
@ -1951,10 +1949,10 @@ void kfd_flush_tlb(struct kfd_process_device *pdd, enum TLB_FLUSH_TYPE type)
|
||||
* only happens when the first queue is created.
|
||||
*/
|
||||
if (pdd->qpd.vmid)
|
||||
amdgpu_amdkfd_flush_gpu_tlb_vmid(dev->kgd,
|
||||
amdgpu_amdkfd_flush_gpu_tlb_vmid(dev->adev,
|
||||
pdd->qpd.vmid);
|
||||
} else {
|
||||
amdgpu_amdkfd_flush_gpu_tlb_pasid(dev->kgd,
|
||||
amdgpu_amdkfd_flush_gpu_tlb_pasid(dev->adev,
|
||||
pdd->process->pasid, type);
|
||||
}
|
||||
}
|
||||
|
||||
@ -118,7 +118,7 @@ int pqm_set_gws(struct process_queue_manager *pqm, unsigned int qid,
|
||||
return ret;
|
||||
|
||||
pqn->q->gws = mem;
|
||||
pdd->qpd.num_gws = gws ? amdgpu_amdkfd_get_num_gws(dev->kgd) : 0;
|
||||
pdd->qpd.num_gws = gws ? dev->adev->gds.gws_size : 0;
|
||||
|
||||
return pqn->q->device->dqm->ops.update_queue(pqn->q->device->dqm,
|
||||
pqn->q, NULL);
|
||||
|
||||
@ -207,7 +207,6 @@ void kfd_smi_event_update_gpu_reset(struct kfd_dev *dev, bool post_reset)
|
||||
void kfd_smi_event_update_thermal_throttling(struct kfd_dev *dev,
|
||||
uint64_t throttle_bitmask)
|
||||
{
|
||||
struct amdgpu_device *adev = (struct amdgpu_device *)dev->kgd;
|
||||
/*
|
||||
* ThermalThrottle msg = throttle_bitmask(8):
|
||||
* thermal_interrupt_count(16):
|
||||
@ -223,14 +222,13 @@ void kfd_smi_event_update_thermal_throttling(struct kfd_dev *dev,
|
||||
|
||||
len = snprintf(fifo_in, sizeof(fifo_in), "%x %llx:%llx\n",
|
||||
KFD_SMI_EVENT_THERMAL_THROTTLE, throttle_bitmask,
|
||||
atomic64_read(&adev->smu.throttle_int_counter));
|
||||
atomic64_read(&dev->adev->smu.throttle_int_counter));
|
||||
|
||||
add_event_to_kfifo(dev, KFD_SMI_EVENT_THERMAL_THROTTLE, fifo_in, len);
|
||||
}
|
||||
|
||||
void kfd_smi_event_update_vmfault(struct kfd_dev *dev, uint16_t pasid)
|
||||
{
|
||||
struct amdgpu_device *adev = (struct amdgpu_device *)dev->kgd;
|
||||
struct amdgpu_task_info task_info;
|
||||
/* VmFault msg = (hex)uint32_pid(8) + :(1) + task name(16) = 25 */
|
||||
/* 1 byte event + 1 byte space + 25 bytes msg + 1 byte \n +
|
||||
@ -243,7 +241,7 @@ void kfd_smi_event_update_vmfault(struct kfd_dev *dev, uint16_t pasid)
|
||||
return;
|
||||
|
||||
memset(&task_info, 0, sizeof(struct amdgpu_task_info));
|
||||
amdgpu_vm_get_task_info(adev, pasid, &task_info);
|
||||
amdgpu_vm_get_task_info(dev->adev, pasid, &task_info);
|
||||
/* Report VM faults from user applications, not retry from kernel */
|
||||
if (!task_info.pid)
|
||||
return;
|
||||
|
||||
@ -193,7 +193,6 @@ svm_range_dma_map(struct svm_range *prange, unsigned long *bitmap,
|
||||
|
||||
for_each_set_bit(gpuidx, bitmap, MAX_GPU_INSTANCE) {
|
||||
struct kfd_process_device *pdd;
|
||||
struct amdgpu_device *adev;
|
||||
|
||||
pr_debug("mapping to gpu idx 0x%x\n", gpuidx);
|
||||
pdd = kfd_process_device_from_gpuidx(p, gpuidx);
|
||||
@ -201,9 +200,8 @@ svm_range_dma_map(struct svm_range *prange, unsigned long *bitmap,
|
||||
pr_debug("failed to find device idx %d\n", gpuidx);
|
||||
return -EINVAL;
|
||||
}
|
||||
adev = (struct amdgpu_device *)pdd->dev->kgd;
|
||||
|
||||
r = svm_range_dma_map_dev(adev, prange, offset, npages,
|
||||
r = svm_range_dma_map_dev(pdd->dev->adev, prange, offset, npages,
|
||||
hmm_pfns, gpuidx);
|
||||
if (r)
|
||||
break;
|
||||
@ -581,7 +579,7 @@ svm_range_get_adev_by_id(struct svm_range *prange, uint32_t gpu_id)
|
||||
return NULL;
|
||||
}
|
||||
|
||||
return (struct amdgpu_device *)pdd->dev->kgd;
|
||||
return pdd->dev->adev;
|
||||
}
|
||||
|
||||
struct kfd_process_device *
|
||||
@ -593,7 +591,7 @@ svm_range_get_pdd_by_adev(struct svm_range *prange, struct amdgpu_device *adev)
|
||||
|
||||
p = container_of(prange->svms, struct kfd_process, svms);
|
||||
|
||||
r = kfd_process_gpuid_from_kgd(p, adev, &gpuid, &gpu_idx);
|
||||
r = kfd_process_gpuid_from_adev(p, adev, &gpuid, &gpu_idx);
|
||||
if (r) {
|
||||
pr_debug("failed to get device id by adev %p\n", adev);
|
||||
return NULL;
|
||||
@ -1053,8 +1051,8 @@ svm_range_get_pte_flags(struct amdgpu_device *adev, struct svm_range *prange,
|
||||
if (domain == SVM_RANGE_VRAM_DOMAIN)
|
||||
bo_adev = amdgpu_ttm_adev(prange->svm_bo->bo->tbo.bdev);
|
||||
|
||||
switch (adev->asic_type) {
|
||||
case CHIP_ARCTURUS:
|
||||
switch (KFD_GC_VERSION(adev->kfd.dev)) {
|
||||
case IP_VERSION(9, 4, 1):
|
||||
if (domain == SVM_RANGE_VRAM_DOMAIN) {
|
||||
if (bo_adev == adev) {
|
||||
mapping_flags |= coherent ?
|
||||
@ -1070,7 +1068,7 @@ svm_range_get_pte_flags(struct amdgpu_device *adev, struct svm_range *prange,
|
||||
AMDGPU_VM_MTYPE_UC : AMDGPU_VM_MTYPE_NC;
|
||||
}
|
||||
break;
|
||||
case CHIP_ALDEBARAN:
|
||||
case IP_VERSION(9, 4, 2):
|
||||
if (domain == SVM_RANGE_VRAM_DOMAIN) {
|
||||
if (bo_adev == adev) {
|
||||
mapping_flags |= coherent ?
|
||||
@ -1129,7 +1127,6 @@ svm_range_unmap_from_gpus(struct svm_range *prange, unsigned long start,
|
||||
DECLARE_BITMAP(bitmap, MAX_GPU_INSTANCE);
|
||||
struct kfd_process_device *pdd;
|
||||
struct dma_fence *fence = NULL;
|
||||
struct amdgpu_device *adev;
|
||||
struct kfd_process *p;
|
||||
uint32_t gpuidx;
|
||||
int r = 0;
|
||||
@ -1145,9 +1142,9 @@ svm_range_unmap_from_gpus(struct svm_range *prange, unsigned long start,
|
||||
pr_debug("failed to find device idx %d\n", gpuidx);
|
||||
return -EINVAL;
|
||||
}
|
||||
adev = (struct amdgpu_device *)pdd->dev->kgd;
|
||||
|
||||
r = svm_range_unmap_from_gpu(adev, drm_priv_to_vm(pdd->drm_priv),
|
||||
r = svm_range_unmap_from_gpu(pdd->dev->adev,
|
||||
drm_priv_to_vm(pdd->drm_priv),
|
||||
start, last, &fence);
|
||||
if (r)
|
||||
break;
|
||||
@ -1159,7 +1156,7 @@ svm_range_unmap_from_gpus(struct svm_range *prange, unsigned long start,
|
||||
if (r)
|
||||
break;
|
||||
}
|
||||
amdgpu_amdkfd_flush_gpu_tlb_pasid((struct kgd_dev *)adev,
|
||||
amdgpu_amdkfd_flush_gpu_tlb_pasid(pdd->dev->adev,
|
||||
p->pasid, TLB_FLUSH_HEAVYWEIGHT);
|
||||
}
|
||||
|
||||
@ -1243,8 +1240,7 @@ svm_range_map_to_gpu(struct amdgpu_device *adev, struct amdgpu_vm *vm,
|
||||
struct kfd_process *p;
|
||||
|
||||
p = container_of(prange->svms, struct kfd_process, svms);
|
||||
amdgpu_amdkfd_flush_gpu_tlb_pasid((struct kgd_dev *)adev,
|
||||
p->pasid, TLB_FLUSH_LEGACY);
|
||||
amdgpu_amdkfd_flush_gpu_tlb_pasid(adev, p->pasid, TLB_FLUSH_LEGACY);
|
||||
}
|
||||
out:
|
||||
return r;
|
||||
@ -1257,7 +1253,6 @@ svm_range_map_to_gpus(struct svm_range *prange, unsigned long offset,
|
||||
{
|
||||
struct kfd_process_device *pdd;
|
||||
struct amdgpu_device *bo_adev;
|
||||
struct amdgpu_device *adev;
|
||||
struct kfd_process *p;
|
||||
struct dma_fence *fence = NULL;
|
||||
uint32_t gpuidx;
|
||||
@ -1276,19 +1271,18 @@ svm_range_map_to_gpus(struct svm_range *prange, unsigned long offset,
|
||||
pr_debug("failed to find device idx %d\n", gpuidx);
|
||||
return -EINVAL;
|
||||
}
|
||||
adev = (struct amdgpu_device *)pdd->dev->kgd;
|
||||
|
||||
pdd = kfd_bind_process_to_device(pdd->dev, p);
|
||||
if (IS_ERR(pdd))
|
||||
return -EINVAL;
|
||||
|
||||
if (bo_adev && adev != bo_adev &&
|
||||
!amdgpu_xgmi_same_hive(adev, bo_adev)) {
|
||||
if (bo_adev && pdd->dev->adev != bo_adev &&
|
||||
!amdgpu_xgmi_same_hive(pdd->dev->adev, bo_adev)) {
|
||||
pr_debug("cannot map to device idx %d\n", gpuidx);
|
||||
continue;
|
||||
}
|
||||
|
||||
r = svm_range_map_to_gpu(adev, drm_priv_to_vm(pdd->drm_priv),
|
||||
r = svm_range_map_to_gpu(pdd->dev->adev, drm_priv_to_vm(pdd->drm_priv),
|
||||
prange, offset, npages, readonly,
|
||||
prange->dma_addr[gpuidx],
|
||||
bo_adev, wait ? &fence : NULL);
|
||||
@ -1322,7 +1316,6 @@ struct svm_validate_context {
|
||||
static int svm_range_reserve_bos(struct svm_validate_context *ctx)
|
||||
{
|
||||
struct kfd_process_device *pdd;
|
||||
struct amdgpu_device *adev;
|
||||
struct amdgpu_vm *vm;
|
||||
uint32_t gpuidx;
|
||||
int r;
|
||||
@ -1334,7 +1327,6 @@ static int svm_range_reserve_bos(struct svm_validate_context *ctx)
|
||||
pr_debug("failed to find device idx %d\n", gpuidx);
|
||||
return -EINVAL;
|
||||
}
|
||||
adev = (struct amdgpu_device *)pdd->dev->kgd;
|
||||
vm = drm_priv_to_vm(pdd->drm_priv);
|
||||
|
||||
ctx->tv[gpuidx].bo = &vm->root.bo->tbo;
|
||||
@ -1356,9 +1348,9 @@ static int svm_range_reserve_bos(struct svm_validate_context *ctx)
|
||||
r = -EINVAL;
|
||||
goto unreserve_out;
|
||||
}
|
||||
adev = (struct amdgpu_device *)pdd->dev->kgd;
|
||||
|
||||
r = amdgpu_vm_validate_pt_bos(adev, drm_priv_to_vm(pdd->drm_priv),
|
||||
r = amdgpu_vm_validate_pt_bos(pdd->dev->adev,
|
||||
drm_priv_to_vm(pdd->drm_priv),
|
||||
svm_range_bo_validate, NULL);
|
||||
if (r) {
|
||||
pr_debug("failed %d validate pt bos\n", r);
|
||||
@ -1381,12 +1373,10 @@ static void svm_range_unreserve_bos(struct svm_validate_context *ctx)
|
||||
static void *kfd_svm_page_owner(struct kfd_process *p, int32_t gpuidx)
|
||||
{
|
||||
struct kfd_process_device *pdd;
|
||||
struct amdgpu_device *adev;
|
||||
|
||||
pdd = kfd_process_device_from_gpuidx(p, gpuidx);
|
||||
adev = (struct amdgpu_device *)pdd->dev->kgd;
|
||||
|
||||
return SVM_ADEV_PGMAP_OWNER(adev);
|
||||
return SVM_ADEV_PGMAP_OWNER(pdd->dev->adev);
|
||||
}
|
||||
|
||||
/*
|
||||
@ -1966,23 +1956,30 @@ svm_range_handle_list_op(struct svm_range_list *svms, struct svm_range *prange)
|
||||
static void svm_range_drain_retry_fault(struct svm_range_list *svms)
|
||||
{
|
||||
struct kfd_process_device *pdd;
|
||||
struct amdgpu_device *adev;
|
||||
struct kfd_process *p;
|
||||
int drain;
|
||||
uint32_t i;
|
||||
|
||||
p = container_of(svms, struct kfd_process, svms);
|
||||
|
||||
restart:
|
||||
drain = atomic_read(&svms->drain_pagefaults);
|
||||
if (!drain)
|
||||
return;
|
||||
|
||||
for_each_set_bit(i, svms->bitmap_supported, p->n_pdds) {
|
||||
pdd = p->pdds[i];
|
||||
if (!pdd)
|
||||
continue;
|
||||
|
||||
pr_debug("drain retry fault gpu %d svms %p\n", i, svms);
|
||||
adev = (struct amdgpu_device *)pdd->dev->kgd;
|
||||
|
||||
amdgpu_ih_wait_on_checkpoint_process(adev, &adev->irq.ih1);
|
||||
amdgpu_ih_wait_on_checkpoint_process(pdd->dev->adev,
|
||||
&pdd->dev->adev->irq.ih1);
|
||||
pr_debug("drain retry fault gpu %d svms 0x%p done\n", i, svms);
|
||||
}
|
||||
if (atomic_cmpxchg(&svms->drain_pagefaults, drain, 0) != drain)
|
||||
goto restart;
|
||||
}
|
||||
|
||||
static void svm_range_deferred_list_work(struct work_struct *work)
|
||||
@ -1990,43 +1987,41 @@ static void svm_range_deferred_list_work(struct work_struct *work)
|
||||
struct svm_range_list *svms;
|
||||
struct svm_range *prange;
|
||||
struct mm_struct *mm;
|
||||
struct kfd_process *p;
|
||||
|
||||
svms = container_of(work, struct svm_range_list, deferred_list_work);
|
||||
pr_debug("enter svms 0x%p\n", svms);
|
||||
|
||||
p = container_of(svms, struct kfd_process, svms);
|
||||
/* Avoid mm is gone when inserting mmu notifier */
|
||||
mm = get_task_mm(p->lead_thread);
|
||||
if (!mm) {
|
||||
pr_debug("svms 0x%p process mm gone\n", svms);
|
||||
return;
|
||||
}
|
||||
retry:
|
||||
mmap_write_lock(mm);
|
||||
|
||||
/* Checking for the need to drain retry faults must be inside
|
||||
* mmap write lock to serialize with munmap notifiers.
|
||||
*/
|
||||
if (unlikely(atomic_read(&svms->drain_pagefaults))) {
|
||||
mmap_write_unlock(mm);
|
||||
svm_range_drain_retry_fault(svms);
|
||||
goto retry;
|
||||
}
|
||||
|
||||
spin_lock(&svms->deferred_list_lock);
|
||||
while (!list_empty(&svms->deferred_range_list)) {
|
||||
prange = list_first_entry(&svms->deferred_range_list,
|
||||
struct svm_range, deferred_list);
|
||||
spin_unlock(&svms->deferred_list_lock);
|
||||
pr_debug("prange 0x%p [0x%lx 0x%lx] op %d\n", prange,
|
||||
prange->start, prange->last, prange->work_item.op);
|
||||
|
||||
mm = prange->work_item.mm;
|
||||
retry:
|
||||
mmap_write_lock(mm);
|
||||
mutex_lock(&svms->lock);
|
||||
|
||||
/* Checking for the need to drain retry faults must be in
|
||||
* mmap write lock to serialize with munmap notifiers.
|
||||
*
|
||||
* Remove from deferred_list must be inside mmap write lock,
|
||||
* otherwise, svm_range_list_lock_and_flush_work may hold mmap
|
||||
* write lock, and continue because deferred_list is empty, then
|
||||
* deferred_list handle is blocked by mmap write lock.
|
||||
*/
|
||||
spin_lock(&svms->deferred_list_lock);
|
||||
if (unlikely(svms->drain_pagefaults)) {
|
||||
svms->drain_pagefaults = false;
|
||||
spin_unlock(&svms->deferred_list_lock);
|
||||
mutex_unlock(&svms->lock);
|
||||
mmap_write_unlock(mm);
|
||||
svm_range_drain_retry_fault(svms);
|
||||
goto retry;
|
||||
}
|
||||
list_del_init(&prange->deferred_list);
|
||||
spin_unlock(&svms->deferred_list_lock);
|
||||
|
||||
pr_debug("prange 0x%p [0x%lx 0x%lx] op %d\n", prange,
|
||||
prange->start, prange->last, prange->work_item.op);
|
||||
|
||||
mutex_lock(&svms->lock);
|
||||
mutex_lock(&prange->migrate_mutex);
|
||||
while (!list_empty(&prange->child_list)) {
|
||||
struct svm_range *pchild;
|
||||
@ -2042,12 +2037,13 @@ retry:
|
||||
|
||||
svm_range_handle_list_op(svms, prange);
|
||||
mutex_unlock(&svms->lock);
|
||||
mmap_write_unlock(mm);
|
||||
|
||||
spin_lock(&svms->deferred_list_lock);
|
||||
}
|
||||
spin_unlock(&svms->deferred_list_lock);
|
||||
|
||||
mmap_write_unlock(mm);
|
||||
mmput(mm);
|
||||
pr_debug("exit svms 0x%p\n", svms);
|
||||
}
|
||||
|
||||
@ -2056,12 +2052,6 @@ svm_range_add_list_work(struct svm_range_list *svms, struct svm_range *prange,
|
||||
struct mm_struct *mm, enum svm_work_list_ops op)
|
||||
{
|
||||
spin_lock(&svms->deferred_list_lock);
|
||||
/* Make sure pending page faults are drained in the deferred worker
|
||||
* before the range is freed to avoid straggler interrupts on
|
||||
* unmapped memory causing "phantom faults".
|
||||
*/
|
||||
if (op == SVM_OP_UNMAP_RANGE)
|
||||
svms->drain_pagefaults = true;
|
||||
/* if prange is on the deferred list */
|
||||
if (!list_empty(&prange->deferred_list)) {
|
||||
pr_debug("update exist prange 0x%p work op %d\n", prange, op);
|
||||
@ -2140,6 +2130,12 @@ svm_range_unmap_from_cpu(struct mm_struct *mm, struct svm_range *prange,
|
||||
pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx] [0x%lx 0x%lx]\n", svms,
|
||||
prange, prange->start, prange->last, start, last);
|
||||
|
||||
/* Make sure pending page faults are drained in the deferred worker
|
||||
* before the range is freed to avoid straggler interrupts on
|
||||
* unmapped memory causing "phantom faults".
|
||||
*/
|
||||
atomic_inc(&svms->drain_pagefaults);
|
||||
|
||||
unmap_parent = start <= prange->start && last >= prange->last;
|
||||
|
||||
list_for_each_entry(pchild, &prange->child_list, child_list) {
|
||||
@ -2301,7 +2297,7 @@ svm_range_best_restore_location(struct svm_range *prange,
|
||||
|
||||
p = container_of(prange->svms, struct kfd_process, svms);
|
||||
|
||||
r = kfd_process_gpuid_from_kgd(p, adev, &gpuid, gpuidx);
|
||||
r = kfd_process_gpuid_from_adev(p, adev, &gpuid, gpuidx);
|
||||
if (r < 0) {
|
||||
pr_debug("failed to get gpuid from kgd\n");
|
||||
return -1;
|
||||
@ -2478,7 +2474,7 @@ svm_range *svm_range_create_unregistered_range(struct amdgpu_device *adev,
|
||||
pr_debug("Failed to create prange in address [0x%llx]\n", addr);
|
||||
return NULL;
|
||||
}
|
||||
if (kfd_process_gpuid_from_kgd(p, adev, &gpuid, &gpuidx)) {
|
||||
if (kfd_process_gpuid_from_adev(p, adev, &gpuid, &gpuidx)) {
|
||||
pr_debug("failed to get gpuid from kgd\n");
|
||||
svm_range_free(prange);
|
||||
return NULL;
|
||||
@ -2545,7 +2541,7 @@ svm_range_count_fault(struct amdgpu_device *adev, struct kfd_process *p,
|
||||
uint32_t gpuid;
|
||||
int r;
|
||||
|
||||
r = kfd_process_gpuid_from_kgd(p, adev, &gpuid, &gpuidx);
|
||||
r = kfd_process_gpuid_from_adev(p, adev, &gpuid, &gpuidx);
|
||||
if (r < 0)
|
||||
return;
|
||||
}
|
||||
@ -2559,20 +2555,13 @@ svm_range_count_fault(struct amdgpu_device *adev, struct kfd_process *p,
|
||||
}
|
||||
|
||||
static bool
|
||||
svm_fault_allowed(struct mm_struct *mm, uint64_t addr, bool write_fault)
|
||||
svm_fault_allowed(struct vm_area_struct *vma, bool write_fault)
|
||||
{
|
||||
unsigned long requested = VM_READ;
|
||||
struct vm_area_struct *vma;
|
||||
|
||||
if (write_fault)
|
||||
requested |= VM_WRITE;
|
||||
|
||||
vma = find_vma(mm, addr << PAGE_SHIFT);
|
||||
if (!vma || (addr << PAGE_SHIFT) < vma->vm_start) {
|
||||
pr_debug("address 0x%llx VMA is removed\n", addr);
|
||||
return true;
|
||||
}
|
||||
|
||||
pr_debug("requested 0x%lx, vma permission flags 0x%lx\n", requested,
|
||||
vma->vm_flags);
|
||||
return (vma->vm_flags & requested) == requested;
|
||||
@ -2590,6 +2579,7 @@ svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid,
|
||||
int32_t best_loc;
|
||||
int32_t gpuidx = MAX_GPU_INSTANCE;
|
||||
bool write_locked = false;
|
||||
struct vm_area_struct *vma;
|
||||
int r = 0;
|
||||
|
||||
if (!KFD_IS_SVM_API_SUPPORTED(adev->kfd.dev)) {
|
||||
@ -2600,7 +2590,7 @@ svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid,
|
||||
p = kfd_lookup_process_by_pasid(pasid);
|
||||
if (!p) {
|
||||
pr_debug("kfd process not founded pasid 0x%x\n", pasid);
|
||||
return -ESRCH;
|
||||
return 0;
|
||||
}
|
||||
if (!p->xnack_enabled) {
|
||||
pr_debug("XNACK not enabled for pasid 0x%x\n", pasid);
|
||||
@ -2611,10 +2601,17 @@ svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid,
|
||||
|
||||
pr_debug("restoring svms 0x%p fault address 0x%llx\n", svms, addr);
|
||||
|
||||
if (atomic_read(&svms->drain_pagefaults)) {
|
||||
pr_debug("draining retry fault, drop fault 0x%llx\n", addr);
|
||||
goto out;
|
||||
}
|
||||
|
||||
/* p->lead_thread is available as kfd_process_wq_release flush the work
|
||||
* before releasing task ref.
|
||||
*/
|
||||
mm = get_task_mm(p->lead_thread);
|
||||
if (!mm) {
|
||||
pr_debug("svms 0x%p failed to get mm\n", svms);
|
||||
r = -ESRCH;
|
||||
goto out;
|
||||
}
|
||||
|
||||
@ -2663,7 +2660,17 @@ retry_write_locked:
|
||||
goto out_unlock_range;
|
||||
}
|
||||
|
||||
if (!svm_fault_allowed(mm, addr, write_fault)) {
|
||||
/* __do_munmap removed VMA, return success as we are handling stale
|
||||
* retry fault.
|
||||
*/
|
||||
vma = find_vma(mm, addr << PAGE_SHIFT);
|
||||
if (!vma || (addr << PAGE_SHIFT) < vma->vm_start) {
|
||||
pr_debug("address 0x%llx VMA is removed\n", addr);
|
||||
r = 0;
|
||||
goto out_unlock_range;
|
||||
}
|
||||
|
||||
if (!svm_fault_allowed(vma, write_fault)) {
|
||||
pr_debug("fault addr 0x%llx no %s permission\n", addr,
|
||||
write_fault ? "write" : "read");
|
||||
r = -EPERM;
|
||||
@ -2741,6 +2748,14 @@ void svm_range_list_fini(struct kfd_process *p)
|
||||
/* Ensure list work is finished before process is destroyed */
|
||||
flush_work(&p->svms.deferred_list_work);
|
||||
|
||||
/*
|
||||
* Ensure no retry fault comes in afterwards, as page fault handler will
|
||||
* not find kfd process and take mm lock to recover fault.
|
||||
*/
|
||||
atomic_inc(&p->svms.drain_pagefaults);
|
||||
svm_range_drain_retry_fault(&p->svms);
|
||||
|
||||
|
||||
list_for_each_entry_safe(prange, next, &p->svms.list, list) {
|
||||
svm_range_unlink(prange);
|
||||
svm_range_remove_notifier(prange);
|
||||
@ -2761,6 +2776,7 @@ int svm_range_list_init(struct kfd_process *p)
|
||||
mutex_init(&svms->lock);
|
||||
INIT_LIST_HEAD(&svms->list);
|
||||
atomic_set(&svms->evicted_ranges, 0);
|
||||
atomic_set(&svms->drain_pagefaults, 0);
|
||||
INIT_DELAYED_WORK(&svms->restore_work, svm_range_restore_work);
|
||||
INIT_WORK(&svms->deferred_list_work, svm_range_deferred_list_work);
|
||||
INIT_LIST_HEAD(&svms->deferred_range_list);
|
||||
@ -2953,7 +2969,6 @@ svm_range_best_prefetch_location(struct svm_range *prange)
|
||||
uint32_t best_loc = prange->prefetch_loc;
|
||||
struct kfd_process_device *pdd;
|
||||
struct amdgpu_device *bo_adev;
|
||||
struct amdgpu_device *adev;
|
||||
struct kfd_process *p;
|
||||
uint32_t gpuidx;
|
||||
|
||||
@ -2981,12 +2996,11 @@ svm_range_best_prefetch_location(struct svm_range *prange)
|
||||
pr_debug("failed to get device by idx 0x%x\n", gpuidx);
|
||||
continue;
|
||||
}
|
||||
adev = (struct amdgpu_device *)pdd->dev->kgd;
|
||||
|
||||
if (adev == bo_adev)
|
||||
if (pdd->dev->adev == bo_adev)
|
||||
continue;
|
||||
|
||||
if (!amdgpu_xgmi_same_hive(adev, bo_adev)) {
|
||||
if (!amdgpu_xgmi_same_hive(pdd->dev->adev, bo_adev)) {
|
||||
best_loc = 0;
|
||||
break;
|
||||
}
|
||||
|
||||
@ -113,7 +113,7 @@ struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev)
|
||||
return device;
|
||||
}
|
||||
|
||||
struct kfd_dev *kfd_device_by_kgd(const struct kgd_dev *kgd)
|
||||
struct kfd_dev *kfd_device_by_adev(const struct amdgpu_device *adev)
|
||||
{
|
||||
struct kfd_topology_device *top_dev;
|
||||
struct kfd_dev *device = NULL;
|
||||
@ -121,7 +121,7 @@ struct kfd_dev *kfd_device_by_kgd(const struct kgd_dev *kgd)
|
||||
down_read(&topology_lock);
|
||||
|
||||
list_for_each_entry(top_dev, &topology_device_list, list)
|
||||
if (top_dev->gpu && top_dev->gpu->kgd == kgd) {
|
||||
if (top_dev->gpu && top_dev->gpu->adev == adev) {
|
||||
device = top_dev->gpu;
|
||||
break;
|
||||
}
|
||||
@ -515,7 +515,7 @@ static ssize_t node_show(struct kobject *kobj, struct attribute *attr,
|
||||
HSA_CAP_WATCH_POINTS_TOTALBITS_MASK);
|
||||
}
|
||||
|
||||
if (dev->gpu->device_info->asic_family == CHIP_TONGA)
|
||||
if (dev->gpu->adev->asic_type == CHIP_TONGA)
|
||||
dev->node_props.capability |=
|
||||
HSA_CAP_AQL_QUEUE_DOUBLE_MAP;
|
||||
|
||||
@ -531,7 +531,7 @@ static ssize_t node_show(struct kobject *kobj, struct attribute *attr,
|
||||
sysfs_show_32bit_prop(buffer, offs, "sdma_fw_version",
|
||||
dev->gpu->sdma_fw_version);
|
||||
sysfs_show_64bit_prop(buffer, offs, "unique_id",
|
||||
amdgpu_amdkfd_get_unique_id(dev->gpu->kgd));
|
||||
dev->gpu->adev->unique_id);
|
||||
|
||||
}
|
||||
|
||||
@ -1106,7 +1106,7 @@ static uint32_t kfd_generate_gpu_id(struct kfd_dev *gpu)
|
||||
if (!gpu)
|
||||
return 0;
|
||||
|
||||
amdgpu_amdkfd_get_local_mem_info(gpu->kgd, &local_mem_info);
|
||||
amdgpu_amdkfd_get_local_mem_info(gpu->adev, &local_mem_info);
|
||||
|
||||
local_mem_size = local_mem_info.local_mem_size_private +
|
||||
local_mem_info.local_mem_size_public;
|
||||
@ -1189,7 +1189,7 @@ static void kfd_fill_mem_clk_max_info(struct kfd_topology_device *dev)
|
||||
* for APUs - If CRAT from ACPI reports more than one bank, then
|
||||
* all the banks will report the same mem_clk_max information
|
||||
*/
|
||||
amdgpu_amdkfd_get_local_mem_info(dev->gpu->kgd, &local_mem_info);
|
||||
amdgpu_amdkfd_get_local_mem_info(dev->gpu->adev, &local_mem_info);
|
||||
|
||||
list_for_each_entry(mem, &dev->mem_props, list)
|
||||
mem->mem_clk_max = local_mem_info.mem_clk_max;
|
||||
@ -1217,8 +1217,7 @@ static void kfd_set_iolink_no_atomics(struct kfd_topology_device *dev,
|
||||
/* set gpu (dev) flags. */
|
||||
} else {
|
||||
if (!dev->gpu->pci_atomic_requested ||
|
||||
dev->gpu->device_info->asic_family ==
|
||||
CHIP_HAWAII)
|
||||
dev->gpu->adev->asic_type == CHIP_HAWAII)
|
||||
link->flags |= CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT |
|
||||
CRAT_IOLINK_FLAGS_NO_ATOMICS_64_BIT;
|
||||
}
|
||||
@ -1239,7 +1238,7 @@ static void kfd_set_iolink_non_coherent(struct kfd_topology_device *to_dev,
|
||||
*/
|
||||
if (inbound_link->iolink_type == CRAT_IOLINK_TYPE_PCIEXPRESS ||
|
||||
(inbound_link->iolink_type == CRAT_IOLINK_TYPE_XGMI &&
|
||||
to_dev->gpu->device_info->asic_family == CHIP_VEGA20)) {
|
||||
KFD_GC_VERSION(to_dev->gpu) == IP_VERSION(9, 4, 0))) {
|
||||
outbound_link->flags |= CRAT_IOLINK_FLAGS_NON_COHERENT;
|
||||
inbound_link->flags |= CRAT_IOLINK_FLAGS_NON_COHERENT;
|
||||
}
|
||||
@ -1286,7 +1285,6 @@ int kfd_topology_add_device(struct kfd_dev *gpu)
|
||||
void *crat_image = NULL;
|
||||
size_t image_size = 0;
|
||||
int proximity_domain;
|
||||
struct amdgpu_device *adev;
|
||||
|
||||
INIT_LIST_HEAD(&temp_topology_device_list);
|
||||
|
||||
@ -1296,10 +1294,8 @@ int kfd_topology_add_device(struct kfd_dev *gpu)
|
||||
|
||||
proximity_domain = atomic_inc_return(&topology_crat_proximity_domain);
|
||||
|
||||
adev = (struct amdgpu_device *)(gpu->kgd);
|
||||
|
||||
/* Include the CPU in xGMI hive if xGMI connected by assigning it the hive ID. */
|
||||
if (gpu->hive_id && adev->gmc.xgmi.connected_to_cpu) {
|
||||
if (gpu->hive_id && gpu->adev->gmc.xgmi.connected_to_cpu) {
|
||||
struct kfd_topology_device *top_dev;
|
||||
|
||||
down_read(&topology_lock);
|
||||
@ -1372,7 +1368,7 @@ int kfd_topology_add_device(struct kfd_dev *gpu)
|
||||
* needed for the topology
|
||||
*/
|
||||
|
||||
amdgpu_amdkfd_get_cu_info(dev->gpu->kgd, &cu_info);
|
||||
amdgpu_amdkfd_get_cu_info(dev->gpu->adev, &cu_info);
|
||||
|
||||
strncpy(dev->node_props.name, gpu->device_info->asic_name,
|
||||
KFD_TOPOLOGY_PUBLIC_NAME_SIZE);
|
||||
@ -1384,33 +1380,32 @@ int kfd_topology_add_device(struct kfd_dev *gpu)
|
||||
dev->node_props.vendor_id = gpu->pdev->vendor;
|
||||
dev->node_props.device_id = gpu->pdev->device;
|
||||
dev->node_props.capability |=
|
||||
((amdgpu_amdkfd_get_asic_rev_id(dev->gpu->kgd) <<
|
||||
HSA_CAP_ASIC_REVISION_SHIFT) &
|
||||
((dev->gpu->adev->rev_id << HSA_CAP_ASIC_REVISION_SHIFT) &
|
||||
HSA_CAP_ASIC_REVISION_MASK);
|
||||
dev->node_props.location_id = pci_dev_id(gpu->pdev);
|
||||
dev->node_props.domain = pci_domain_nr(gpu->pdev->bus);
|
||||
dev->node_props.max_engine_clk_fcompute =
|
||||
amdgpu_amdkfd_get_max_engine_clock_in_mhz(dev->gpu->kgd);
|
||||
amdgpu_amdkfd_get_max_engine_clock_in_mhz(dev->gpu->adev);
|
||||
dev->node_props.max_engine_clk_ccompute =
|
||||
cpufreq_quick_get_max(0) / 1000;
|
||||
dev->node_props.drm_render_minor =
|
||||
gpu->shared_resources.drm_render_minor;
|
||||
|
||||
dev->node_props.hive_id = gpu->hive_id;
|
||||
dev->node_props.num_sdma_engines = gpu->device_info->num_sdma_engines;
|
||||
dev->node_props.num_sdma_engines = kfd_get_num_sdma_engines(gpu);
|
||||
dev->node_props.num_sdma_xgmi_engines =
|
||||
gpu->device_info->num_xgmi_sdma_engines;
|
||||
kfd_get_num_xgmi_sdma_engines(gpu);
|
||||
dev->node_props.num_sdma_queues_per_engine =
|
||||
gpu->device_info->num_sdma_queues_per_engine;
|
||||
dev->node_props.num_gws = (dev->gpu->gws &&
|
||||
dev->gpu->dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) ?
|
||||
amdgpu_amdkfd_get_num_gws(dev->gpu->kgd) : 0;
|
||||
dev->gpu->adev->gds.gws_size : 0;
|
||||
dev->node_props.num_cp_queues = get_cp_queues_num(dev->gpu->dqm);
|
||||
|
||||
kfd_fill_mem_clk_max_info(dev);
|
||||
kfd_fill_iolink_non_crat_info(dev);
|
||||
|
||||
switch (dev->gpu->device_info->asic_family) {
|
||||
switch (dev->gpu->adev->asic_type) {
|
||||
case CHIP_KAVERI:
|
||||
case CHIP_HAWAII:
|
||||
case CHIP_TONGA:
|
||||
@ -1429,30 +1424,14 @@ int kfd_topology_add_device(struct kfd_dev *gpu)
|
||||
HSA_CAP_DOORBELL_TYPE_TOTALBITS_SHIFT) &
|
||||
HSA_CAP_DOORBELL_TYPE_TOTALBITS_MASK);
|
||||
break;
|
||||
case CHIP_VEGA10:
|
||||
case CHIP_VEGA12:
|
||||
case CHIP_VEGA20:
|
||||
case CHIP_RAVEN:
|
||||
case CHIP_RENOIR:
|
||||
case CHIP_ARCTURUS:
|
||||
case CHIP_ALDEBARAN:
|
||||
case CHIP_NAVI10:
|
||||
case CHIP_NAVI12:
|
||||
case CHIP_NAVI14:
|
||||
case CHIP_SIENNA_CICHLID:
|
||||
case CHIP_NAVY_FLOUNDER:
|
||||
case CHIP_VANGOGH:
|
||||
case CHIP_DIMGREY_CAVEFISH:
|
||||
case CHIP_BEIGE_GOBY:
|
||||
case CHIP_YELLOW_CARP:
|
||||
case CHIP_CYAN_SKILLFISH:
|
||||
dev->node_props.capability |= ((HSA_CAP_DOORBELL_TYPE_2_0 <<
|
||||
HSA_CAP_DOORBELL_TYPE_TOTALBITS_SHIFT) &
|
||||
HSA_CAP_DOORBELL_TYPE_TOTALBITS_MASK);
|
||||
break;
|
||||
default:
|
||||
WARN(1, "Unexpected ASIC family %u",
|
||||
dev->gpu->device_info->asic_family);
|
||||
if (KFD_GC_VERSION(dev->gpu) >= IP_VERSION(9, 0, 1))
|
||||
dev->node_props.capability |= ((HSA_CAP_DOORBELL_TYPE_2_0 <<
|
||||
HSA_CAP_DOORBELL_TYPE_TOTALBITS_SHIFT) &
|
||||
HSA_CAP_DOORBELL_TYPE_TOTALBITS_MASK);
|
||||
else
|
||||
WARN(1, "Unexpected ASIC family %u",
|
||||
dev->gpu->adev->asic_type);
|
||||
}
|
||||
|
||||
/*
|
||||
@ -1469,7 +1448,7 @@ int kfd_topology_add_device(struct kfd_dev *gpu)
|
||||
* because it doesn't consider masked out CUs
|
||||
* max_waves_per_simd: Carrizo reports wrong max_waves_per_simd
|
||||
*/
|
||||
if (dev->gpu->device_info->asic_family == CHIP_CARRIZO) {
|
||||
if (dev->gpu->adev->asic_type == CHIP_CARRIZO) {
|
||||
dev->node_props.simd_count =
|
||||
cu_info.simd_per_cu * cu_info.cu_active_number;
|
||||
dev->node_props.max_waves_per_simd = 10;
|
||||
@ -1477,16 +1456,17 @@ int kfd_topology_add_device(struct kfd_dev *gpu)
|
||||
|
||||
/* kfd only concerns sram ecc on GFX and HBM ecc on UMC */
|
||||
dev->node_props.capability |=
|
||||
((adev->ras_enabled & BIT(AMDGPU_RAS_BLOCK__GFX)) != 0) ?
|
||||
((dev->gpu->adev->ras_enabled & BIT(AMDGPU_RAS_BLOCK__GFX)) != 0) ?
|
||||
HSA_CAP_SRAM_EDCSUPPORTED : 0;
|
||||
dev->node_props.capability |= ((adev->ras_enabled & BIT(AMDGPU_RAS_BLOCK__UMC)) != 0) ?
|
||||
dev->node_props.capability |=
|
||||
((dev->gpu->adev->ras_enabled & BIT(AMDGPU_RAS_BLOCK__UMC)) != 0) ?
|
||||
HSA_CAP_MEM_EDCSUPPORTED : 0;
|
||||
|
||||
if (adev->asic_type != CHIP_VEGA10)
|
||||
dev->node_props.capability |= (adev->ras_enabled != 0) ?
|
||||
if (KFD_GC_VERSION(dev->gpu) != IP_VERSION(9, 0, 1))
|
||||
dev->node_props.capability |= (dev->gpu->adev->ras_enabled != 0) ?
|
||||
HSA_CAP_RASEVENTNOTIFY : 0;
|
||||
|
||||
if (KFD_IS_SVM_API_SUPPORTED(adev->kfd.dev))
|
||||
if (KFD_IS_SVM_API_SUPPORTED(dev->gpu->adev->kfd.dev))
|
||||
dev->node_props.capability |= HSA_CAP_SVMAPI_SUPPORTED;
|
||||
|
||||
kfd_debug_print_topology();
|
||||
|
||||
@ -25,38 +25,11 @@
|
||||
|
||||
#include <linux/types.h>
|
||||
#include <linux/list.h>
|
||||
#include <linux/kfd_sysfs.h>
|
||||
#include "kfd_crat.h"
|
||||
|
||||
#define KFD_TOPOLOGY_PUBLIC_NAME_SIZE 32
|
||||
|
||||
#define HSA_CAP_HOT_PLUGGABLE 0x00000001
|
||||
#define HSA_CAP_ATS_PRESENT 0x00000002
|
||||
#define HSA_CAP_SHARED_WITH_GRAPHICS 0x00000004
|
||||
#define HSA_CAP_QUEUE_SIZE_POW2 0x00000008
|
||||
#define HSA_CAP_QUEUE_SIZE_32BIT 0x00000010
|
||||
#define HSA_CAP_QUEUE_IDLE_EVENT 0x00000020
|
||||
#define HSA_CAP_VA_LIMIT 0x00000040
|
||||
#define HSA_CAP_WATCH_POINTS_SUPPORTED 0x00000080
|
||||
#define HSA_CAP_WATCH_POINTS_TOTALBITS_MASK 0x00000f00
|
||||
#define HSA_CAP_WATCH_POINTS_TOTALBITS_SHIFT 8
|
||||
#define HSA_CAP_DOORBELL_TYPE_TOTALBITS_MASK 0x00003000
|
||||
#define HSA_CAP_DOORBELL_TYPE_TOTALBITS_SHIFT 12
|
||||
|
||||
#define HSA_CAP_DOORBELL_TYPE_PRE_1_0 0x0
|
||||
#define HSA_CAP_DOORBELL_TYPE_1_0 0x1
|
||||
#define HSA_CAP_DOORBELL_TYPE_2_0 0x2
|
||||
#define HSA_CAP_AQL_QUEUE_DOUBLE_MAP 0x00004000
|
||||
|
||||
#define HSA_CAP_RESERVED_WAS_SRAM_EDCSUPPORTED 0x00080000 /* Old buggy user mode depends on this being 0 */
|
||||
#define HSA_CAP_MEM_EDCSUPPORTED 0x00100000
|
||||
#define HSA_CAP_RASEVENTNOTIFY 0x00200000
|
||||
#define HSA_CAP_ASIC_REVISION_MASK 0x03c00000
|
||||
#define HSA_CAP_ASIC_REVISION_SHIFT 22
|
||||
#define HSA_CAP_SRAM_EDCSUPPORTED 0x04000000
|
||||
#define HSA_CAP_SVMAPI_SUPPORTED 0x08000000
|
||||
#define HSA_CAP_FLAGS_COHERENTHOSTACCESS 0x10000000
|
||||
#define HSA_CAP_RESERVED 0xe00f8000
|
||||
|
||||
struct kfd_node_properties {
|
||||
uint64_t hive_id;
|
||||
uint32_t cpu_cores_count;
|
||||
@ -93,17 +66,6 @@ struct kfd_node_properties {
|
||||
char name[KFD_TOPOLOGY_PUBLIC_NAME_SIZE];
|
||||
};
|
||||
|
||||
#define HSA_MEM_HEAP_TYPE_SYSTEM 0
|
||||
#define HSA_MEM_HEAP_TYPE_FB_PUBLIC 1
|
||||
#define HSA_MEM_HEAP_TYPE_FB_PRIVATE 2
|
||||
#define HSA_MEM_HEAP_TYPE_GPU_GDS 3
|
||||
#define HSA_MEM_HEAP_TYPE_GPU_LDS 4
|
||||
#define HSA_MEM_HEAP_TYPE_GPU_SCRATCH 5
|
||||
|
||||
#define HSA_MEM_FLAGS_HOT_PLUGGABLE 0x00000001
|
||||
#define HSA_MEM_FLAGS_NON_VOLATILE 0x00000002
|
||||
#define HSA_MEM_FLAGS_RESERVED 0xfffffffc
|
||||
|
||||
struct kfd_mem_properties {
|
||||
struct list_head list;
|
||||
uint32_t heap_type;
|
||||
@ -116,12 +78,6 @@ struct kfd_mem_properties {
|
||||
struct attribute attr;
|
||||
};
|
||||
|
||||
#define HSA_CACHE_TYPE_DATA 0x00000001
|
||||
#define HSA_CACHE_TYPE_INSTRUCTION 0x00000002
|
||||
#define HSA_CACHE_TYPE_CPU 0x00000004
|
||||
#define HSA_CACHE_TYPE_HSACU 0x00000008
|
||||
#define HSA_CACHE_TYPE_RESERVED 0xfffffff0
|
||||
|
||||
struct kfd_cache_properties {
|
||||
struct list_head list;
|
||||
uint32_t processor_id_low;
|
||||
|
||||
@ -51,6 +51,7 @@
|
||||
#include <drm/drm_hdcp.h>
|
||||
#endif
|
||||
#include "amdgpu_pm.h"
|
||||
#include "amdgpu_atombios.h"
|
||||
|
||||
#include "amd_shared.h"
|
||||
#include "amdgpu_dm_irq.h"
|
||||
@ -789,8 +790,7 @@ static void dm_dmub_outbox1_low_irq(void *interrupt_params)
|
||||
plink = adev->dm.dc->links[notify.link_index];
|
||||
if (plink) {
|
||||
plink->hpd_status =
|
||||
notify.hpd_status ==
|
||||
DP_HPD_PLUG ? true : false;
|
||||
notify.hpd_status == DP_HPD_PLUG;
|
||||
}
|
||||
}
|
||||
queue_work(adev->dm.delayed_hpd_wq, &dmub_hpd_wrk->handle_hpd_work);
|
||||
@ -1455,6 +1455,12 @@ static int amdgpu_dm_init(struct amdgpu_device *adev)
|
||||
|
||||
init_data.flags.power_down_display_on_boot = true;
|
||||
|
||||
if (check_seamless_boot_capability(adev)) {
|
||||
init_data.flags.power_down_display_on_boot = false;
|
||||
init_data.flags.allow_seamless_boot_optimization = true;
|
||||
DRM_INFO("Seamless boot condition check passed\n");
|
||||
}
|
||||
|
||||
INIT_LIST_HEAD(&adev->dm.da_list);
|
||||
/* Display Core create. */
|
||||
adev->dm.dc = dc_create(&init_data);
|
||||
@ -1479,8 +1485,10 @@ static int amdgpu_dm_init(struct amdgpu_device *adev)
|
||||
if (amdgpu_dc_debug_mask & DC_DISABLE_STUTTER)
|
||||
adev->dm.dc->debug.disable_stutter = true;
|
||||
|
||||
if (amdgpu_dc_debug_mask & DC_DISABLE_DSC)
|
||||
if (amdgpu_dc_debug_mask & DC_DISABLE_DSC) {
|
||||
adev->dm.dc->debug.disable_dsc = true;
|
||||
adev->dm.dc->debug.disable_dsc_edp = true;
|
||||
}
|
||||
|
||||
if (amdgpu_dc_debug_mask & DC_DISABLE_CLOCK_GATING)
|
||||
adev->dm.dc->debug.disable_clock_gate = true;
|
||||
@ -2303,14 +2311,6 @@ static enum dc_status amdgpu_dm_commit_zero_streams(struct dc *dc)
|
||||
goto fail;
|
||||
}
|
||||
|
||||
|
||||
res = dc_validate_global_state(dc, context, false);
|
||||
|
||||
if (res != DC_OK) {
|
||||
DRM_ERROR("%s:resource validation failed, dc_status:%d\n", __func__, res);
|
||||
goto fail;
|
||||
}
|
||||
|
||||
res = dc_commit_state(dc, context);
|
||||
|
||||
fail:
|
||||
@ -2561,6 +2561,22 @@ static int dm_resume(void *handle)
|
||||
if (amdgpu_in_reset(adev)) {
|
||||
dc_state = dm->cached_dc_state;
|
||||
|
||||
/*
|
||||
* The dc->current_state is backed up into dm->cached_dc_state
|
||||
* before we commit 0 streams.
|
||||
*
|
||||
* DC will clear link encoder assignments on the real state
|
||||
* but the changes won't propagate over to the copy we made
|
||||
* before the 0 streams commit.
|
||||
*
|
||||
* DC expects that link encoder assignments are *not* valid
|
||||
* when committing a state, so as a workaround it needs to be
|
||||
* cleared here.
|
||||
*/
|
||||
link_enc_cfg_init(dm->dc, dc_state);
|
||||
|
||||
amdgpu_dm_outbox_init(adev);
|
||||
|
||||
r = dm_dmub_hw_init(adev);
|
||||
if (r)
|
||||
DRM_ERROR("DMUB interface failed to initialize: status=%d\n", r);
|
||||
@ -2572,20 +2588,11 @@ static int dm_resume(void *handle)
|
||||
|
||||
for (i = 0; i < dc_state->stream_count; i++) {
|
||||
dc_state->streams[i]->mode_changed = true;
|
||||
for (j = 0; j < dc_state->stream_status->plane_count; j++) {
|
||||
dc_state->stream_status->plane_states[j]->update_flags.raw
|
||||
for (j = 0; j < dc_state->stream_status[i].plane_count; j++) {
|
||||
dc_state->stream_status[i].plane_states[j]->update_flags.raw
|
||||
= 0xffffffff;
|
||||
}
|
||||
}
|
||||
#if defined(CONFIG_DRM_AMD_DC_DCN)
|
||||
/*
|
||||
* Resource allocation happens for link encoders for newer ASIC in
|
||||
* dc_validate_global_state, so we need to revalidate it.
|
||||
*
|
||||
* This shouldn't fail (it passed once before), so warn if it does.
|
||||
*/
|
||||
WARN_ON(dc_validate_global_state(dm->dc, dc_state, false) != DC_OK);
|
||||
#endif
|
||||
|
||||
WARN_ON(!dc_commit_state(dm->dc, dc_state));
|
||||
|
||||
@ -3909,6 +3916,9 @@ static int amdgpu_dm_backlight_set_level(struct amdgpu_display_manager *dm,
|
||||
caps = dm->backlight_caps[bl_idx];
|
||||
|
||||
dm->brightness[bl_idx] = user_brightness;
|
||||
/* update scratch register */
|
||||
if (bl_idx == 0)
|
||||
amdgpu_atombios_scratch_regs_set_backlight_level(dm->adev, dm->brightness[bl_idx]);
|
||||
brightness = convert_brightness_from_user(&caps, dm->brightness[bl_idx]);
|
||||
link = (struct dc_link *)dm->backlight_link[bl_idx];
|
||||
|
||||
@ -4251,6 +4261,14 @@ static int amdgpu_dm_initialize_drm_device(struct amdgpu_device *adev)
|
||||
|
||||
}
|
||||
|
||||
/*
|
||||
* Disable vblank IRQs aggressively for power-saving.
|
||||
*
|
||||
* TODO: Fix vblank control helpers to delay PSR entry to allow this when PSR
|
||||
* is also supported.
|
||||
*/
|
||||
adev_to_drm(adev)->vblank_disable_immediate = !psr_feature_enabled;
|
||||
|
||||
/* Software is initialized. Now we can register interrupt handlers. */
|
||||
switch (adev->asic_type) {
|
||||
#if defined(CONFIG_DRM_AMD_DC_SI)
|
||||
@ -6036,7 +6054,8 @@ static void update_dsc_caps(struct amdgpu_dm_connector *aconnector,
|
||||
{
|
||||
stream->timing.flags.DSC = 0;
|
||||
|
||||
if (aconnector->dc_link && sink->sink_signal == SIGNAL_TYPE_DISPLAY_PORT) {
|
||||
if (aconnector->dc_link && (sink->sink_signal == SIGNAL_TYPE_DISPLAY_PORT ||
|
||||
sink->sink_signal == SIGNAL_TYPE_EDP)) {
|
||||
dc_dsc_parse_dsc_dpcd(aconnector->dc_link->ctx->dc,
|
||||
aconnector->dc_link->dpcd_caps.dsc_caps.dsc_basic_caps.raw,
|
||||
aconnector->dc_link->dpcd_caps.dsc_caps.dsc_branch_decoder_caps.raw,
|
||||
@ -6044,6 +6063,64 @@ static void update_dsc_caps(struct amdgpu_dm_connector *aconnector,
|
||||
}
|
||||
}
|
||||
|
||||
static void apply_dsc_policy_for_edp(struct amdgpu_dm_connector *aconnector,
|
||||
struct dc_sink *sink, struct dc_stream_state *stream,
|
||||
struct dsc_dec_dpcd_caps *dsc_caps,
|
||||
uint32_t max_dsc_target_bpp_limit_override)
|
||||
{
|
||||
const struct dc_link_settings *verified_link_cap = NULL;
|
||||
uint32_t link_bw_in_kbps;
|
||||
uint32_t edp_min_bpp_x16, edp_max_bpp_x16;
|
||||
struct dc *dc = sink->ctx->dc;
|
||||
struct dc_dsc_bw_range bw_range = {0};
|
||||
struct dc_dsc_config dsc_cfg = {0};
|
||||
|
||||
verified_link_cap = dc_link_get_link_cap(stream->link);
|
||||
link_bw_in_kbps = dc_link_bandwidth_kbps(stream->link, verified_link_cap);
|
||||
edp_min_bpp_x16 = 8 * 16;
|
||||
edp_max_bpp_x16 = 8 * 16;
|
||||
|
||||
if (edp_max_bpp_x16 > dsc_caps->edp_max_bits_per_pixel)
|
||||
edp_max_bpp_x16 = dsc_caps->edp_max_bits_per_pixel;
|
||||
|
||||
if (edp_max_bpp_x16 < edp_min_bpp_x16)
|
||||
edp_min_bpp_x16 = edp_max_bpp_x16;
|
||||
|
||||
if (dc_dsc_compute_bandwidth_range(dc->res_pool->dscs[0],
|
||||
dc->debug.dsc_min_slice_height_override,
|
||||
edp_min_bpp_x16, edp_max_bpp_x16,
|
||||
dsc_caps,
|
||||
&stream->timing,
|
||||
&bw_range)) {
|
||||
|
||||
if (bw_range.max_kbps < link_bw_in_kbps) {
|
||||
if (dc_dsc_compute_config(dc->res_pool->dscs[0],
|
||||
dsc_caps,
|
||||
dc->debug.dsc_min_slice_height_override,
|
||||
max_dsc_target_bpp_limit_override,
|
||||
0,
|
||||
&stream->timing,
|
||||
&dsc_cfg)) {
|
||||
stream->timing.dsc_cfg = dsc_cfg;
|
||||
stream->timing.flags.DSC = 1;
|
||||
stream->timing.dsc_cfg.bits_per_pixel = edp_max_bpp_x16;
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (dc_dsc_compute_config(dc->res_pool->dscs[0],
|
||||
dsc_caps,
|
||||
dc->debug.dsc_min_slice_height_override,
|
||||
max_dsc_target_bpp_limit_override,
|
||||
link_bw_in_kbps,
|
||||
&stream->timing,
|
||||
&dsc_cfg)) {
|
||||
stream->timing.dsc_cfg = dsc_cfg;
|
||||
stream->timing.flags.DSC = 1;
|
||||
}
|
||||
}
|
||||
|
||||
static void apply_dsc_policy_for_stream(struct amdgpu_dm_connector *aconnector,
|
||||
struct dc_sink *sink, struct dc_stream_state *stream,
|
||||
struct dsc_dec_dpcd_caps *dsc_caps)
|
||||
@ -6051,6 +6128,7 @@ static void apply_dsc_policy_for_stream(struct amdgpu_dm_connector *aconnector,
|
||||
struct drm_connector *drm_connector = &aconnector->base;
|
||||
uint32_t link_bandwidth_kbps;
|
||||
uint32_t max_dsc_target_bpp_limit_override = 0;
|
||||
struct dc *dc = sink->ctx->dc;
|
||||
|
||||
link_bandwidth_kbps = dc_link_bandwidth_kbps(aconnector->dc_link,
|
||||
dc_link_get_link_cap(aconnector->dc_link));
|
||||
@ -6063,7 +6141,12 @@ static void apply_dsc_policy_for_stream(struct amdgpu_dm_connector *aconnector,
|
||||
dc_dsc_policy_set_enable_dsc_when_not_needed(
|
||||
aconnector->dsc_settings.dsc_force_enable == DSC_CLK_FORCE_ENABLE);
|
||||
|
||||
if (aconnector->dc_link && sink->sink_signal == SIGNAL_TYPE_DISPLAY_PORT) {
|
||||
if (aconnector->dc_link && sink->sink_signal == SIGNAL_TYPE_EDP && !dc->debug.disable_dsc_edp &&
|
||||
dc->caps.edp_dsc_support && aconnector->dsc_settings.dsc_force_enable != DSC_CLK_FORCE_DISABLE) {
|
||||
|
||||
apply_dsc_policy_for_edp(aconnector, sink, stream, dsc_caps, max_dsc_target_bpp_limit_override);
|
||||
|
||||
} else if (aconnector->dc_link && sink->sink_signal == SIGNAL_TYPE_DISPLAY_PORT) {
|
||||
|
||||
if (dc_dsc_compute_config(aconnector->dc_link->ctx->dc->res_pool->dscs[0],
|
||||
dsc_caps,
|
||||
@ -10759,8 +10842,10 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
|
||||
trace_amdgpu_dm_atomic_check_begin(state);
|
||||
|
||||
ret = drm_atomic_helper_check_modeset(dev, state);
|
||||
if (ret)
|
||||
if (ret) {
|
||||
DRM_DEBUG_DRIVER("drm_atomic_helper_check_modeset() failed\n");
|
||||
goto fail;
|
||||
}
|
||||
|
||||
/* Check connector changes */
|
||||
for_each_oldnew_connector_in_state(state, connector, old_con_state, new_con_state, i) {
|
||||
@ -10776,6 +10861,7 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
|
||||
|
||||
new_crtc_state = drm_atomic_get_crtc_state(state, new_con_state->crtc);
|
||||
if (IS_ERR(new_crtc_state)) {
|
||||
DRM_DEBUG_DRIVER("drm_atomic_get_crtc_state() failed\n");
|
||||
ret = PTR_ERR(new_crtc_state);
|
||||
goto fail;
|
||||
}
|
||||
@ -10790,8 +10876,10 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
|
||||
for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
|
||||
if (drm_atomic_crtc_needs_modeset(new_crtc_state)) {
|
||||
ret = add_affected_mst_dsc_crtcs(state, crtc);
|
||||
if (ret)
|
||||
if (ret) {
|
||||
DRM_DEBUG_DRIVER("add_affected_mst_dsc_crtcs() failed\n");
|
||||
goto fail;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -10806,19 +10894,25 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
|
||||
continue;
|
||||
|
||||
ret = amdgpu_dm_verify_lut_sizes(new_crtc_state);
|
||||
if (ret)
|
||||
if (ret) {
|
||||
DRM_DEBUG_DRIVER("amdgpu_dm_verify_lut_sizes() failed\n");
|
||||
goto fail;
|
||||
}
|
||||
|
||||
if (!new_crtc_state->enable)
|
||||
continue;
|
||||
|
||||
ret = drm_atomic_add_affected_connectors(state, crtc);
|
||||
if (ret)
|
||||
if (ret) {
|
||||
DRM_DEBUG_DRIVER("drm_atomic_add_affected_connectors() failed\n");
|
||||
goto fail;
|
||||
}
|
||||
|
||||
ret = drm_atomic_add_affected_planes(state, crtc);
|
||||
if (ret)
|
||||
if (ret) {
|
||||
DRM_DEBUG_DRIVER("drm_atomic_add_affected_planes() failed\n");
|
||||
goto fail;
|
||||
}
|
||||
|
||||
if (dm_old_crtc_state->dsc_force_changed)
|
||||
new_crtc_state->mode_changed = true;
|
||||
@ -10855,6 +10949,7 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
|
||||
|
||||
if (IS_ERR(new_plane_state)) {
|
||||
ret = PTR_ERR(new_plane_state);
|
||||
DRM_DEBUG_DRIVER("new_plane_state is BAD\n");
|
||||
goto fail;
|
||||
}
|
||||
}
|
||||
@ -10867,8 +10962,10 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
|
||||
new_plane_state,
|
||||
false,
|
||||
&lock_and_validation_needed);
|
||||
if (ret)
|
||||
if (ret) {
|
||||
DRM_DEBUG_DRIVER("dm_update_plane_state() failed\n");
|
||||
goto fail;
|
||||
}
|
||||
}
|
||||
|
||||
/* Disable all crtcs which require disable */
|
||||
@ -10878,8 +10975,10 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
|
||||
new_crtc_state,
|
||||
false,
|
||||
&lock_and_validation_needed);
|
||||
if (ret)
|
||||
if (ret) {
|
||||
DRM_DEBUG_DRIVER("DISABLE: dm_update_crtc_state() failed\n");
|
||||
goto fail;
|
||||
}
|
||||
}
|
||||
|
||||
/* Enable all crtcs which require enable */
|
||||
@ -10889,8 +10988,10 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
|
||||
new_crtc_state,
|
||||
true,
|
||||
&lock_and_validation_needed);
|
||||
if (ret)
|
||||
if (ret) {
|
||||
DRM_DEBUG_DRIVER("ENABLE: dm_update_crtc_state() failed\n");
|
||||
goto fail;
|
||||
}
|
||||
}
|
||||
|
||||
/* Add new/modified planes */
|
||||
@ -10900,20 +11001,26 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
|
||||
new_plane_state,
|
||||
true,
|
||||
&lock_and_validation_needed);
|
||||
if (ret)
|
||||
if (ret) {
|
||||
DRM_DEBUG_DRIVER("dm_update_plane_state() failed\n");
|
||||
goto fail;
|
||||
}
|
||||
}
|
||||
|
||||
/* Run this here since we want to validate the streams we created */
|
||||
ret = drm_atomic_helper_check_planes(dev, state);
|
||||
if (ret)
|
||||
if (ret) {
|
||||
DRM_DEBUG_DRIVER("drm_atomic_helper_check_planes() failed\n");
|
||||
goto fail;
|
||||
}
|
||||
|
||||
/* Check cursor planes scaling */
|
||||
for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) {
|
||||
ret = dm_check_crtc_cursor(state, crtc, new_crtc_state);
|
||||
if (ret)
|
||||
if (ret) {
|
||||
DRM_DEBUG_DRIVER("dm_check_crtc_cursor() failed\n");
|
||||
goto fail;
|
||||
}
|
||||
}
|
||||
|
||||
if (state->legacy_cursor_update) {
|
||||
@ -11000,20 +11107,28 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
|
||||
*/
|
||||
if (lock_and_validation_needed) {
|
||||
ret = dm_atomic_get_state(state, &dm_state);
|
||||
if (ret)
|
||||
if (ret) {
|
||||
DRM_DEBUG_DRIVER("dm_atomic_get_state() failed\n");
|
||||
goto fail;
|
||||
}
|
||||
|
||||
ret = do_aquire_global_lock(dev, state);
|
||||
if (ret)
|
||||
if (ret) {
|
||||
DRM_DEBUG_DRIVER("do_aquire_global_lock() failed\n");
|
||||
goto fail;
|
||||
}
|
||||
|
||||
#if defined(CONFIG_DRM_AMD_DC_DCN)
|
||||
if (!compute_mst_dsc_configs_for_state(state, dm_state->context, vars))
|
||||
if (!compute_mst_dsc_configs_for_state(state, dm_state->context, vars)) {
|
||||
DRM_DEBUG_DRIVER("compute_mst_dsc_configs_for_state() failed\n");
|
||||
goto fail;
|
||||
}
|
||||
|
||||
ret = dm_update_mst_vcpi_slots_for_dsc(state, dm_state->context, vars);
|
||||
if (ret)
|
||||
if (ret) {
|
||||
DRM_DEBUG_DRIVER("dm_update_mst_vcpi_slots_for_dsc() failed\n");
|
||||
goto fail;
|
||||
}
|
||||
#endif
|
||||
|
||||
/*
|
||||
@ -11023,12 +11138,13 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
|
||||
* to get stuck in an infinite loop and hang eventually.
|
||||
*/
|
||||
ret = drm_dp_mst_atomic_check(state);
|
||||
if (ret)
|
||||
if (ret) {
|
||||
DRM_DEBUG_DRIVER("drm_dp_mst_atomic_check() failed\n");
|
||||
goto fail;
|
||||
status = dc_validate_global_state(dc, dm_state->context, false);
|
||||
}
|
||||
status = dc_validate_global_state(dc, dm_state->context, true);
|
||||
if (status != DC_OK) {
|
||||
drm_dbg_atomic(dev,
|
||||
"DC global validation failure: %s (%d)",
|
||||
DRM_DEBUG_DRIVER("DC global validation failure: %s (%d)",
|
||||
dc_status_to_str(status), status);
|
||||
ret = -EINVAL;
|
||||
goto fail;
|
||||
@ -11529,3 +11645,24 @@ int amdgpu_dm_process_dmub_aux_transfer_sync(bool is_cmd_aux, struct dc_context
|
||||
ctx, DMUB_ASYNC_TO_SYNC_ACCESS_SUCCESS,
|
||||
(uint32_t *)operation_result);
|
||||
}
|
||||
|
||||
/*
|
||||
* Check whether seamless boot is supported.
|
||||
*
|
||||
* So far we only support seamless boot on CHIP_VANGOGH.
|
||||
* If everything goes well, we may consider expanding
|
||||
* seamless boot to other ASICs.
|
||||
*/
|
||||
bool check_seamless_boot_capability(struct amdgpu_device *adev)
|
||||
{
|
||||
switch (adev->asic_type) {
|
||||
case CHIP_VANGOGH:
|
||||
if (!adev->mman.keep_stolen_vga_memory)
|
||||
return true;
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
@ -731,4 +731,7 @@ extern const struct drm_encoder_helper_funcs amdgpu_dm_encoder_helper_funcs;
|
||||
int amdgpu_dm_process_dmub_aux_transfer_sync(bool is_cmd_aux,
|
||||
struct dc_context *ctx, unsigned int link_index,
|
||||
void *payload, void *operation_result);
|
||||
|
||||
bool check_seamless_boot_capability(struct amdgpu_device *adev);
|
||||
|
||||
#endif /* __AMDGPU_DM_H__ */
|
||||
|
||||
@ -584,7 +584,7 @@ bool dm_helpers_dp_write_dsc_enable(
|
||||
ret = drm_dp_dpcd_write(aconnector->dsc_aux, DP_DSC_ENABLE, &enable_dsc, 1);
|
||||
}
|
||||
|
||||
if (stream->signal == SIGNAL_TYPE_DISPLAY_PORT) {
|
||||
if (stream->signal == SIGNAL_TYPE_DISPLAY_PORT || stream->signal == SIGNAL_TYPE_EDP) {
|
||||
ret = dm_helpers_dp_write_dpcd(ctx, stream->link, DP_DSC_ENABLE, &enable_dsc, 1);
|
||||
DC_LOG_DC("Send DSC %s to sst display\n", enable_dsc ? "enable" : "disable");
|
||||
}
|
||||
|
||||
@ -2995,7 +2995,7 @@ static bool bios_parser2_construct(
|
||||
&bp->object_info_tbl.revision);
|
||||
|
||||
if (bp->object_info_tbl.revision.major == 1
|
||||
&& bp->object_info_tbl.revision.minor >= 4) {
|
||||
&& bp->object_info_tbl.revision.minor == 4) {
|
||||
struct display_object_info_table_v1_4 *tbl_v1_4;
|
||||
|
||||
tbl_v1_4 = GET_IMAGE(struct display_object_info_table_v1_4,
|
||||
@ -3004,8 +3004,10 @@ static bool bios_parser2_construct(
|
||||
return false;
|
||||
|
||||
bp->object_info_tbl.v1_4 = tbl_v1_4;
|
||||
} else
|
||||
} else {
|
||||
ASSERT(0);
|
||||
return false;
|
||||
}
|
||||
|
||||
dal_firmware_parser_init_cmd_tbl(bp);
|
||||
dal_bios_parser_init_cmd_tbl_helper2(&bp->cmd_helper, dce_version);
|
||||
|
||||
@ -763,7 +763,7 @@ unsigned int get_highest_allowed_voltage_level(uint32_t chip_family, uint32_t hw
|
||||
return 4;
|
||||
}
|
||||
|
||||
bool dcn_validate_bandwidth(
|
||||
bool dcn10_validate_bandwidth(
|
||||
struct dc *dc,
|
||||
struct dc_state *context,
|
||||
bool fast_validate)
|
||||
|
||||
@ -308,8 +308,7 @@ void dc_destroy_clk_mgr(struct clk_mgr *clk_mgr_base)
|
||||
case FAMILY_NV:
|
||||
if (ASICREV_IS_SIENNA_CICHLID_P(clk_mgr_base->ctx->asic_id.hw_internal_rev)) {
|
||||
dcn3_clk_mgr_destroy(clk_mgr);
|
||||
}
|
||||
if (ASICREV_IS_DIMGREY_CAVEFISH_P(clk_mgr_base->ctx->asic_id.hw_internal_rev)) {
|
||||
} else if (ASICREV_IS_DIMGREY_CAVEFISH_P(clk_mgr_base->ctx->asic_id.hw_internal_rev)) {
|
||||
dcn3_clk_mgr_destroy(clk_mgr);
|
||||
}
|
||||
if (ASICREV_IS_BEIGE_GOBY_P(clk_mgr_base->ctx->asic_id.hw_internal_rev)) {
|
||||
|
||||
@ -66,7 +66,7 @@
|
||||
#define TO_CLK_MGR_DCN31(clk_mgr)\
|
||||
container_of(clk_mgr, struct clk_mgr_dcn31, base)
|
||||
|
||||
int dcn31_get_active_display_cnt_wa(
|
||||
static int dcn31_get_active_display_cnt_wa(
|
||||
struct dc *dc,
|
||||
struct dc_state *context)
|
||||
{
|
||||
@ -118,7 +118,7 @@ static void dcn31_disable_otg_wa(struct clk_mgr *clk_mgr_base, bool disable)
|
||||
}
|
||||
}
|
||||
|
||||
static void dcn31_update_clocks(struct clk_mgr *clk_mgr_base,
|
||||
void dcn31_update_clocks(struct clk_mgr *clk_mgr_base,
|
||||
struct dc_state *context,
|
||||
bool safe_to_lower)
|
||||
{
|
||||
@ -284,7 +284,7 @@ static void dcn31_enable_pme_wa(struct clk_mgr *clk_mgr_base)
|
||||
dcn31_smu_enable_pme_wa(clk_mgr);
|
||||
}
|
||||
|
||||
static void dcn31_init_clocks(struct clk_mgr *clk_mgr)
|
||||
void dcn31_init_clocks(struct clk_mgr *clk_mgr)
|
||||
{
|
||||
memset(&(clk_mgr->clks), 0, sizeof(struct dc_clocks));
|
||||
// Assumption is that boot state always supports pstate
|
||||
@ -294,7 +294,7 @@ static void dcn31_init_clocks(struct clk_mgr *clk_mgr)
|
||||
clk_mgr->clks.zstate_support = DCN_ZSTATE_SUPPORT_UNKNOWN;
|
||||
}
|
||||
|
||||
static bool dcn31_are_clock_states_equal(struct dc_clocks *a,
|
||||
bool dcn31_are_clock_states_equal(struct dc_clocks *a,
|
||||
struct dc_clocks *b)
|
||||
{
|
||||
if (a->dispclk_khz != b->dispclk_khz)
|
||||
|
||||
@ -39,6 +39,13 @@ struct clk_mgr_dcn31 {
|
||||
struct dcn31_smu_watermark_set smu_wm_set;
|
||||
};
|
||||
|
||||
bool dcn31_are_clock_states_equal(struct dc_clocks *a,
|
||||
struct dc_clocks *b);
|
||||
void dcn31_init_clocks(struct clk_mgr *clk_mgr);
|
||||
void dcn31_update_clocks(struct clk_mgr *clk_mgr_base,
|
||||
struct dc_state *context,
|
||||
bool safe_to_lower);
|
||||
|
||||
void dcn31_clk_mgr_construct(struct dc_context *ctx,
|
||||
struct clk_mgr_dcn31 *clk_mgr,
|
||||
struct pp_smu_funcs *pp_smu,
|
||||
|
||||
@ -221,9 +221,9 @@ static bool create_links(
|
||||
link = link_create(&link_init_params);
|
||||
|
||||
if (link) {
|
||||
dc->links[dc->link_count] = link;
|
||||
link->dc = dc;
|
||||
++dc->link_count;
|
||||
dc->links[dc->link_count] = link;
|
||||
link->dc = dc;
|
||||
++dc->link_count;
|
||||
}
|
||||
}
|
||||
|
||||
@ -808,6 +808,10 @@ void dc_stream_set_static_screen_params(struct dc *dc,
|
||||
|
||||
static void dc_destruct(struct dc *dc)
|
||||
{
|
||||
// reset link encoder assignment table on destruct
|
||||
if (dc->res_pool->funcs->link_encs_assign)
|
||||
link_enc_cfg_init(dc, dc->current_state);
|
||||
|
||||
if (dc->current_state) {
|
||||
dc_release_state(dc->current_state);
|
||||
dc->current_state = NULL;
|
||||
@ -1016,8 +1020,6 @@ static bool dc_construct(struct dc *dc,
|
||||
goto fail;
|
||||
}
|
||||
|
||||
dc_resource_state_construct(dc, dc->current_state);
|
||||
|
||||
if (!create_links(dc, init_params->num_virtual_links))
|
||||
goto fail;
|
||||
|
||||
@ -1027,8 +1029,7 @@ static bool dc_construct(struct dc *dc,
|
||||
if (!create_link_encoders(dc))
|
||||
goto fail;
|
||||
|
||||
/* Initialise DIG link encoder resource tracking variables. */
|
||||
link_enc_cfg_init(dc, dc->current_state);
|
||||
dc_resource_state_construct(dc, dc->current_state);
|
||||
|
||||
return true;
|
||||
|
||||
@ -1830,6 +1831,19 @@ bool dc_commit_state(struct dc *dc, struct dc_state *context)
|
||||
dc_stream_log(dc, stream);
|
||||
}
|
||||
|
||||
/*
|
||||
* Previous validation was perfomred with fast_validation = true and
|
||||
* the full DML state required for hardware programming was skipped.
|
||||
*
|
||||
* Re-validate here to calculate these parameters / watermarks.
|
||||
*/
|
||||
result = dc_validate_global_state(dc, context, false);
|
||||
if (result != DC_OK) {
|
||||
DC_LOG_ERROR("DC commit global validation failure: %s (%d)",
|
||||
dc_status_to_str(result), result);
|
||||
return result;
|
||||
}
|
||||
|
||||
result = dc_commit_state_no_check(dc, context);
|
||||
|
||||
return (result == DC_OK);
|
||||
@ -2870,7 +2884,8 @@ static void commit_planes_for_stream(struct dc *dc,
|
||||
#endif
|
||||
|
||||
if ((update_type != UPDATE_TYPE_FAST) && stream->update_flags.bits.dsc_changed)
|
||||
if (top_pipe_to_program->stream_res.tg->funcs->lock_doublebuffer_enable) {
|
||||
if (top_pipe_to_program &&
|
||||
top_pipe_to_program->stream_res.tg->funcs->lock_doublebuffer_enable) {
|
||||
if (should_use_dmub_lock(stream->link)) {
|
||||
union dmub_hw_lock_flags hw_locks = { 0 };
|
||||
struct dmub_hw_lock_inst_flags inst_flags = { 0 };
|
||||
|
||||
@ -270,10 +270,10 @@ bool dc_link_detect_sink(struct dc_link *link, enum dc_connection_type *type)
|
||||
|
||||
/* Link may not have physical HPD pin. */
|
||||
if (link->ep_type != DISPLAY_ENDPOINT_PHY) {
|
||||
if (link->hpd_status)
|
||||
*type = dc_connection_single;
|
||||
else
|
||||
if (link->is_hpd_pending || !link->hpd_status)
|
||||
*type = dc_connection_none;
|
||||
else
|
||||
*type = dc_connection_single;
|
||||
|
||||
return true;
|
||||
}
|
||||
@ -1999,6 +1999,57 @@ static enum dc_status enable_link_dp_mst(
|
||||
return enable_link_dp(state, pipe_ctx);
|
||||
}
|
||||
|
||||
void dc_link_blank_all_dp_displays(struct dc *dc)
|
||||
{
|
||||
unsigned int i;
|
||||
uint8_t dpcd_power_state = '\0';
|
||||
enum dc_status status = DC_ERROR_UNEXPECTED;
|
||||
|
||||
for (i = 0; i < dc->link_count; i++) {
|
||||
if ((dc->links[i]->connector_signal != SIGNAL_TYPE_DISPLAY_PORT) ||
|
||||
(dc->links[i]->priv == NULL) || (dc->links[i]->local_sink == NULL))
|
||||
continue;
|
||||
|
||||
/* DP 2.0 spec requires that we read LTTPR caps first */
|
||||
dp_retrieve_lttpr_cap(dc->links[i]);
|
||||
/* if any of the displays are lit up turn them off */
|
||||
status = core_link_read_dpcd(dc->links[i], DP_SET_POWER,
|
||||
&dpcd_power_state, sizeof(dpcd_power_state));
|
||||
|
||||
if (status == DC_OK && dpcd_power_state == DP_POWER_STATE_D0)
|
||||
dc_link_blank_dp_stream(dc->links[i], true);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
void dc_link_blank_dp_stream(struct dc_link *link, bool hw_init)
|
||||
{
|
||||
unsigned int j;
|
||||
struct dc *dc = link->ctx->dc;
|
||||
enum signal_type signal = link->connector_signal;
|
||||
|
||||
if ((signal == SIGNAL_TYPE_EDP) ||
|
||||
(signal == SIGNAL_TYPE_DISPLAY_PORT)) {
|
||||
if (link->ep_type == DISPLAY_ENDPOINT_PHY &&
|
||||
link->link_enc->funcs->get_dig_frontend &&
|
||||
link->link_enc->funcs->is_dig_enabled(link->link_enc)) {
|
||||
unsigned int fe = link->link_enc->funcs->get_dig_frontend(link->link_enc);
|
||||
|
||||
if (fe != ENGINE_ID_UNKNOWN)
|
||||
for (j = 0; j < dc->res_pool->stream_enc_count; j++) {
|
||||
if (fe == dc->res_pool->stream_enc[j]->id) {
|
||||
dc->res_pool->stream_enc[j]->funcs->dp_blank(link,
|
||||
dc->res_pool->stream_enc[j]);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ((!link->wa_flags.dp_keep_receiver_powered) || hw_init)
|
||||
dp_receiver_power_ctrl(link, false);
|
||||
}
|
||||
}
|
||||
|
||||
static bool get_ext_hdmi_settings(struct pipe_ctx *pipe_ctx,
|
||||
enum engine_id eng_id,
|
||||
struct ext_hdmi_settings *settings)
|
||||
@ -2946,7 +2997,7 @@ bool dc_link_set_psr_allow_active(struct dc_link *link, const bool *allow_active
|
||||
link->psr_settings.psr_power_opt = *power_opts;
|
||||
|
||||
if (psr != NULL && link->psr_settings.psr_feature_enabled && psr->funcs->psr_set_power_opt)
|
||||
psr->funcs->psr_set_power_opt(psr, link->psr_settings.psr_power_opt);
|
||||
psr->funcs->psr_set_power_opt(psr, link->psr_settings.psr_power_opt, panel_inst);
|
||||
}
|
||||
|
||||
/* Enable or Disable PSR */
|
||||
@ -3913,9 +3964,6 @@ static void update_psp_stream_config(struct pipe_ctx *pipe_ctx, bool dpms_off)
|
||||
struct cp_psp *cp_psp = &pipe_ctx->stream->ctx->cp_psp;
|
||||
#if defined(CONFIG_DRM_AMD_DC_DCN)
|
||||
struct link_encoder *link_enc = NULL;
|
||||
struct dc_state *state = pipe_ctx->stream->ctx->dc->current_state;
|
||||
struct link_enc_assignment link_enc_assign;
|
||||
int i;
|
||||
#endif
|
||||
|
||||
if (cp_psp && cp_psp->funcs.update_stream_config) {
|
||||
@ -3943,18 +3991,15 @@ static void update_psp_stream_config(struct pipe_ctx *pipe_ctx, bool dpms_off)
|
||||
pipe_ctx->stream->ctx->dc,
|
||||
pipe_ctx->stream);
|
||||
}
|
||||
ASSERT(link_enc);
|
||||
|
||||
// Initialize PHY ID with ABCDE - 01234 mapping except when it is B0
|
||||
config.phy_idx = link_enc->transmitter - TRANSMITTER_UNIPHY_A;
|
||||
|
||||
//look up the link_enc_assignment for the current pipe_ctx
|
||||
for (i = 0; i < state->stream_count; i++) {
|
||||
if (pipe_ctx->stream == state->streams[i]) {
|
||||
link_enc_assign = state->res_ctx.link_enc_cfg_ctx.link_enc_assignments[i];
|
||||
}
|
||||
}
|
||||
// Add flag to guard new A0 DIG mapping
|
||||
if (pipe_ctx->stream->ctx->dc->enable_c20_dtm_b0 == true) {
|
||||
config.dig_be = link_enc_assign.eng_id;
|
||||
if (pipe_ctx->stream->ctx->dc->enable_c20_dtm_b0 == true &&
|
||||
pipe_ctx->stream->link->dc->ctx->dce_version == DCN_VERSION_3_1) {
|
||||
config.dig_be = link_enc->preferred_engine;
|
||||
config.dio_output_type = pipe_ctx->stream->link->ep_type;
|
||||
config.dio_output_idx = link_enc->transmitter - TRANSMITTER_UNIPHY_A;
|
||||
} else {
|
||||
@ -3966,10 +4011,8 @@ static void update_psp_stream_config(struct pipe_ctx *pipe_ctx, bool dpms_off)
|
||||
if (pipe_ctx->stream->ctx->dc->enable_c20_dtm_b0 == true &&
|
||||
link_enc->ctx->asic_id.hw_internal_rev == YELLOW_CARP_B0) {
|
||||
if (pipe_ctx->stream->link->ep_type == DISPLAY_ENDPOINT_USB4_DPIA) {
|
||||
link_enc = link_enc_assign.stream->link_enc;
|
||||
|
||||
// enum ID 1-4 maps to DPIA PHY ID 0-3
|
||||
config.phy_idx = link_enc_assign.ep_id.link_id.enum_id - ENUM_ID_1;
|
||||
config.phy_idx = pipe_ctx->stream->link->link_id.enum_id - ENUM_ID_1;
|
||||
} else { // for non DPIA mode over B0, ABCDE maps to 01564
|
||||
|
||||
switch (link_enc->transmitter) {
|
||||
@ -4242,7 +4285,8 @@ void core_link_enable_stream(
|
||||
/* eDP lit up by bios already, no need to enable again. */
|
||||
if (pipe_ctx->stream->signal == SIGNAL_TYPE_EDP &&
|
||||
apply_edp_fast_boot_optimization &&
|
||||
!pipe_ctx->stream->timing.flags.DSC) {
|
||||
!pipe_ctx->stream->timing.flags.DSC &&
|
||||
!pipe_ctx->next_odm_pipe) {
|
||||
pipe_ctx->stream->dpms_off = false;
|
||||
#if defined(CONFIG_DRM_AMD_DC_HDCP)
|
||||
update_psp_stream_config(pipe_ctx, false);
|
||||
@ -4749,6 +4793,8 @@ bool dc_link_should_enable_fec(const struct dc_link *link)
|
||||
link->local_sink &&
|
||||
link->local_sink->edid_caps.panel_patch.disable_fec) ||
|
||||
(link->connector_signal == SIGNAL_TYPE_EDP
|
||||
// enable FEC for EDP if DSC is supported
|
||||
&& link->dpcd_caps.dsc_caps.dsc_basic_caps.fields.dsc_support.DSC_SUPPORT == false
|
||||
))
|
||||
is_fec_disable = true;
|
||||
|
||||
|
||||
@ -626,7 +626,7 @@ bool dal_ddc_submit_aux_command(struct ddc_service *ddc,
|
||||
do {
|
||||
struct aux_payload current_payload;
|
||||
bool is_end_of_payload = (retrieved + DEFAULT_AUX_MAX_DATA_SIZE) >=
|
||||
payload->length ? true : false;
|
||||
payload->length;
|
||||
uint32_t payload_length = is_end_of_payload ?
|
||||
payload->length - retrieved : DEFAULT_AUX_MAX_DATA_SIZE;
|
||||
|
||||
|
||||
@ -430,7 +430,7 @@ enum dc_status dpcd_set_link_settings(
|
||||
status = core_link_write_dpcd(link, DP_LANE_COUNT_SET,
|
||||
&lane_count_set.raw, 1);
|
||||
|
||||
if (link->dpcd_caps.dpcd_rev.raw >= DPCD_REV_14 &&
|
||||
if (link->dpcd_caps.dpcd_rev.raw >= DPCD_REV_13 &&
|
||||
lt_settings->link_settings.use_link_rate_set == true) {
|
||||
rate = 0;
|
||||
/* WA for some MUX chips that will power down with eDP and lose supported
|
||||
@ -3346,6 +3346,148 @@ bool decide_edp_link_settings(struct dc_link *link, struct dc_link_settings *lin
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool decide_edp_link_settings_with_dsc(struct dc_link *link,
|
||||
struct dc_link_settings *link_setting,
|
||||
uint32_t req_bw,
|
||||
enum dc_link_rate max_link_rate)
|
||||
{
|
||||
struct dc_link_settings initial_link_setting;
|
||||
struct dc_link_settings current_link_setting;
|
||||
uint32_t link_bw;
|
||||
|
||||
unsigned int policy = 0;
|
||||
|
||||
policy = link->ctx->dc->debug.force_dsc_edp_policy;
|
||||
if (max_link_rate == LINK_RATE_UNKNOWN)
|
||||
max_link_rate = link->verified_link_cap.link_rate;
|
||||
/*
|
||||
* edp_supported_link_rates_count is only valid for eDP v1.4 or higher.
|
||||
* Per VESA eDP spec, "The DPCD revision for eDP v1.4 is 13h"
|
||||
*/
|
||||
if ((link->dpcd_caps.dpcd_rev.raw < DPCD_REV_13 ||
|
||||
link->dpcd_caps.edp_supported_link_rates_count == 0)) {
|
||||
/* for DSC enabled case, we search for minimum lane count */
|
||||
memset(&initial_link_setting, 0, sizeof(initial_link_setting));
|
||||
initial_link_setting.lane_count = LANE_COUNT_ONE;
|
||||
initial_link_setting.link_rate = LINK_RATE_LOW;
|
||||
initial_link_setting.link_spread = LINK_SPREAD_DISABLED;
|
||||
initial_link_setting.use_link_rate_set = false;
|
||||
initial_link_setting.link_rate_set = 0;
|
||||
current_link_setting = initial_link_setting;
|
||||
if (req_bw > dc_link_bandwidth_kbps(link, &link->verified_link_cap))
|
||||
return false;
|
||||
|
||||
/* search for the minimum link setting that:
|
||||
* 1. is supported according to the link training result
|
||||
* 2. could support the b/w requested by the timing
|
||||
*/
|
||||
while (current_link_setting.link_rate <=
|
||||
max_link_rate) {
|
||||
link_bw = dc_link_bandwidth_kbps(
|
||||
link,
|
||||
¤t_link_setting);
|
||||
if (req_bw <= link_bw) {
|
||||
*link_setting = current_link_setting;
|
||||
return true;
|
||||
}
|
||||
if (policy) {
|
||||
/* minimize lane */
|
||||
if (current_link_setting.link_rate < max_link_rate) {
|
||||
current_link_setting.link_rate =
|
||||
increase_link_rate(
|
||||
current_link_setting.link_rate);
|
||||
} else {
|
||||
if (current_link_setting.lane_count <
|
||||
link->verified_link_cap.lane_count) {
|
||||
current_link_setting.lane_count =
|
||||
increase_lane_count(
|
||||
current_link_setting.lane_count);
|
||||
current_link_setting.link_rate = initial_link_setting.link_rate;
|
||||
} else
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
/* minimize link rate */
|
||||
if (current_link_setting.lane_count <
|
||||
link->verified_link_cap.lane_count) {
|
||||
current_link_setting.lane_count =
|
||||
increase_lane_count(
|
||||
current_link_setting.lane_count);
|
||||
} else {
|
||||
current_link_setting.link_rate =
|
||||
increase_link_rate(
|
||||
current_link_setting.link_rate);
|
||||
current_link_setting.lane_count =
|
||||
initial_link_setting.lane_count;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/* if optimize edp link is supported */
|
||||
memset(&initial_link_setting, 0, sizeof(initial_link_setting));
|
||||
initial_link_setting.lane_count = LANE_COUNT_ONE;
|
||||
initial_link_setting.link_rate = link->dpcd_caps.edp_supported_link_rates[0];
|
||||
initial_link_setting.link_spread = LINK_SPREAD_DISABLED;
|
||||
initial_link_setting.use_link_rate_set = true;
|
||||
initial_link_setting.link_rate_set = 0;
|
||||
current_link_setting = initial_link_setting;
|
||||
|
||||
/* search for the minimum link setting that:
|
||||
* 1. is supported according to the link training result
|
||||
* 2. could support the b/w requested by the timing
|
||||
*/
|
||||
while (current_link_setting.link_rate <=
|
||||
max_link_rate) {
|
||||
link_bw = dc_link_bandwidth_kbps(
|
||||
link,
|
||||
¤t_link_setting);
|
||||
if (req_bw <= link_bw) {
|
||||
*link_setting = current_link_setting;
|
||||
return true;
|
||||
}
|
||||
if (policy) {
|
||||
/* minimize lane */
|
||||
if (current_link_setting.link_rate_set <
|
||||
link->dpcd_caps.edp_supported_link_rates_count
|
||||
&& current_link_setting.link_rate < max_link_rate) {
|
||||
current_link_setting.link_rate_set++;
|
||||
current_link_setting.link_rate =
|
||||
link->dpcd_caps.edp_supported_link_rates[current_link_setting.link_rate_set];
|
||||
} else {
|
||||
if (current_link_setting.lane_count < link->verified_link_cap.lane_count) {
|
||||
current_link_setting.lane_count =
|
||||
increase_lane_count(
|
||||
current_link_setting.lane_count);
|
||||
current_link_setting.link_rate_set = initial_link_setting.link_rate_set;
|
||||
current_link_setting.link_rate =
|
||||
link->dpcd_caps.edp_supported_link_rates[current_link_setting.link_rate_set];
|
||||
} else
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
/* minimize link rate */
|
||||
if (current_link_setting.lane_count <
|
||||
link->verified_link_cap.lane_count) {
|
||||
current_link_setting.lane_count =
|
||||
increase_lane_count(
|
||||
current_link_setting.lane_count);
|
||||
} else {
|
||||
if (current_link_setting.link_rate_set < link->dpcd_caps.edp_supported_link_rates_count) {
|
||||
current_link_setting.link_rate_set++;
|
||||
current_link_setting.link_rate =
|
||||
link->dpcd_caps.edp_supported_link_rates[current_link_setting.link_rate_set];
|
||||
current_link_setting.lane_count =
|
||||
initial_link_setting.lane_count;
|
||||
} else
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool decide_mst_link_settings(const struct dc_link *link, struct dc_link_settings *link_setting)
|
||||
{
|
||||
*link_setting = link->verified_link_cap;
|
||||
@ -3380,7 +3522,25 @@ void decide_link_settings(struct dc_stream_state *stream,
|
||||
if (decide_mst_link_settings(link, link_setting))
|
||||
return;
|
||||
} else if (link->connector_signal == SIGNAL_TYPE_EDP) {
|
||||
if (decide_edp_link_settings(link, link_setting, req_bw))
|
||||
/* enable edp link optimization for DSC eDP case */
|
||||
if (stream->timing.flags.DSC) {
|
||||
enum dc_link_rate max_link_rate = LINK_RATE_UNKNOWN;
|
||||
|
||||
if (link->ctx->dc->debug.force_dsc_edp_policy) {
|
||||
/* calculate link max link rate cap*/
|
||||
struct dc_link_settings tmp_link_setting;
|
||||
struct dc_crtc_timing tmp_timing = stream->timing;
|
||||
uint32_t orig_req_bw;
|
||||
|
||||
tmp_link_setting.link_rate = LINK_RATE_UNKNOWN;
|
||||
tmp_timing.flags.DSC = 0;
|
||||
orig_req_bw = dc_bandwidth_in_kbps_from_timing(&tmp_timing);
|
||||
decide_edp_link_settings(link, &tmp_link_setting, orig_req_bw);
|
||||
max_link_rate = tmp_link_setting.link_rate;
|
||||
}
|
||||
if (decide_edp_link_settings_with_dsc(link, link_setting, req_bw, max_link_rate))
|
||||
return;
|
||||
} else if (decide_edp_link_settings(link, link_setting, req_bw))
|
||||
return;
|
||||
} else if (decide_dp_link_settings(link, link_setting, req_bw))
|
||||
return;
|
||||
@ -4454,7 +4614,7 @@ bool dp_retrieve_lttpr_cap(struct dc_link *link)
|
||||
lttpr_dpcd_data,
|
||||
sizeof(lttpr_dpcd_data));
|
||||
if (status != DC_OK) {
|
||||
dm_error("%s: Read LTTPR caps data failed.\n", __func__);
|
||||
DC_LOG_DP2("%s: Read LTTPR caps data failed.\n", __func__);
|
||||
return false;
|
||||
}
|
||||
|
||||
@ -5885,7 +6045,10 @@ bool is_edp_ilr_optimization_required(struct dc_link *link, struct dc_crtc_timin
|
||||
|
||||
req_bw = dc_bandwidth_in_kbps_from_timing(crtc_timing);
|
||||
|
||||
decide_edp_link_settings(link, &link_setting, req_bw);
|
||||
if (!crtc_timing->flags.DSC)
|
||||
decide_edp_link_settings(link, &link_setting, req_bw);
|
||||
else
|
||||
decide_edp_link_settings_with_dsc(link, &link_setting, req_bw, LINK_RATE_UNKNOWN);
|
||||
|
||||
if (link->dpcd_caps.edp_supported_link_rates[link_rate_set] != link_setting.link_rate ||
|
||||
lane_count_set.bits.LANE_COUNT_SET != link_setting.lane_count) {
|
||||
|
||||
@ -94,17 +94,17 @@ static enum link_training_result dpia_configure_link(struct dc_link *link,
|
||||
lt_settings);
|
||||
|
||||
status = dpcd_configure_channel_coding(link, lt_settings);
|
||||
if (status != DC_OK && !link->hpd_status)
|
||||
if (status != DC_OK && link->is_hpd_pending)
|
||||
return LINK_TRAINING_ABORT;
|
||||
|
||||
/* Configure lttpr mode */
|
||||
status = dpcd_configure_lttpr_mode(link, lt_settings);
|
||||
if (status != DC_OK && !link->hpd_status)
|
||||
if (status != DC_OK && link->is_hpd_pending)
|
||||
return LINK_TRAINING_ABORT;
|
||||
|
||||
/* Set link rate, lane count and spread. */
|
||||
status = dpcd_set_link_settings(link, lt_settings);
|
||||
if (status != DC_OK && !link->hpd_status)
|
||||
if (status != DC_OK && link->is_hpd_pending)
|
||||
return LINK_TRAINING_ABORT;
|
||||
|
||||
if (link->preferred_training_settings.fec_enable)
|
||||
@ -112,7 +112,7 @@ static enum link_training_result dpia_configure_link(struct dc_link *link,
|
||||
else
|
||||
fec_enable = true;
|
||||
status = dp_set_fec_ready(link, fec_enable);
|
||||
if (status != DC_OK && !link->hpd_status)
|
||||
if (status != DC_OK && link->is_hpd_pending)
|
||||
return LINK_TRAINING_ABORT;
|
||||
|
||||
return LINK_TRAINING_SUCCESS;
|
||||
@ -388,7 +388,7 @@ static enum link_training_result dpia_training_cr_non_transparent(struct dc_link
|
||||
}
|
||||
|
||||
/* Abort link training if clock recovery failed due to HPD unplug. */
|
||||
if (!link->hpd_status)
|
||||
if (link->is_hpd_pending)
|
||||
result = LINK_TRAINING_ABORT;
|
||||
|
||||
DC_LOG_HW_LINK_TRAINING("%s\n DPIA(%d) clock recovery\n"
|
||||
@ -490,7 +490,7 @@ static enum link_training_result dpia_training_cr_transparent(struct dc_link *li
|
||||
}
|
||||
|
||||
/* Abort link training if clock recovery failed due to HPD unplug. */
|
||||
if (!link->hpd_status)
|
||||
if (link->is_hpd_pending)
|
||||
result = LINK_TRAINING_ABORT;
|
||||
|
||||
DC_LOG_HW_LINK_TRAINING("%s\n DPIA(%d) clock recovery\n"
|
||||
@ -675,7 +675,7 @@ static enum link_training_result dpia_training_eq_non_transparent(struct dc_link
|
||||
}
|
||||
|
||||
/* Abort link training if equalization failed due to HPD unplug. */
|
||||
if (!link->hpd_status)
|
||||
if (link->is_hpd_pending)
|
||||
result = LINK_TRAINING_ABORT;
|
||||
|
||||
DC_LOG_HW_LINK_TRAINING("%s\n DPIA(%d) equalization\n"
|
||||
@ -758,7 +758,7 @@ static enum link_training_result dpia_training_eq_transparent(struct dc_link *li
|
||||
}
|
||||
|
||||
/* Abort link training if equalization failed due to HPD unplug. */
|
||||
if (!link->hpd_status)
|
||||
if (link->is_hpd_pending)
|
||||
result = LINK_TRAINING_ABORT;
|
||||
|
||||
DC_LOG_HW_LINK_TRAINING("%s\n DPIA(%d) equalization\n"
|
||||
@ -892,10 +892,10 @@ static void dpia_training_abort(struct dc_link *link, uint32_t hop)
|
||||
__func__,
|
||||
link->link_id.enum_id - ENUM_ID_1,
|
||||
link->lttpr_mode,
|
||||
link->hpd_status);
|
||||
link->is_hpd_pending);
|
||||
|
||||
/* Abandon clean-up if sink unplugged. */
|
||||
if (!link->hpd_status)
|
||||
if (link->is_hpd_pending)
|
||||
return;
|
||||
|
||||
if (hop != DPRX)
|
||||
|
||||
@ -118,7 +118,10 @@ static void remove_link_enc_assignment(
|
||||
*/
|
||||
if (get_stream_using_link_enc(state, eng_id) == NULL)
|
||||
state->res_ctx.link_enc_cfg_ctx.link_enc_avail[eng_idx] = eng_id;
|
||||
|
||||
stream->link_enc = NULL;
|
||||
state->res_ctx.link_enc_cfg_ctx.link_enc_assignments[i].eng_id = ENGINE_ID_UNKNOWN;
|
||||
state->res_ctx.link_enc_cfg_ctx.link_enc_assignments[i].stream = NULL;
|
||||
break;
|
||||
}
|
||||
}
|
||||
@ -148,6 +151,7 @@ static void add_link_enc_assignment(
|
||||
.ep_type = stream->link->ep_type},
|
||||
.eng_id = eng_id,
|
||||
.stream = stream};
|
||||
dc_stream_retain(stream);
|
||||
state->res_ctx.link_enc_cfg_ctx.link_enc_avail[eng_idx] = ENGINE_ID_UNKNOWN;
|
||||
stream->link_enc = stream->ctx->dc->res_pool->link_encoders[eng_idx];
|
||||
break;
|
||||
@ -227,7 +231,7 @@ static struct link_encoder *get_link_enc_used_by_link(
|
||||
.link_id = link->link_id,
|
||||
.ep_type = link->ep_type};
|
||||
|
||||
for (i = 0; i < state->stream_count; i++) {
|
||||
for (i = 0; i < MAX_PIPES; i++) {
|
||||
struct link_enc_assignment assignment = state->res_ctx.link_enc_cfg_ctx.link_enc_assignments[i];
|
||||
|
||||
if (assignment.valid == true && are_ep_ids_equal(&assignment.ep_id, &ep_id))
|
||||
@ -237,28 +241,18 @@ static struct link_encoder *get_link_enc_used_by_link(
|
||||
return link_enc;
|
||||
}
|
||||
/* Clear all link encoder assignments. */
|
||||
static void clear_enc_assignments(struct dc_state *state)
|
||||
static void clear_enc_assignments(const struct dc *dc, struct dc_state *state)
|
||||
{
|
||||
int i;
|
||||
enum engine_id eng_id;
|
||||
struct dc_stream_state *stream;
|
||||
|
||||
for (i = 0; i < MAX_PIPES; i++) {
|
||||
state->res_ctx.link_enc_cfg_ctx.link_enc_assignments[i].valid = false;
|
||||
eng_id = state->res_ctx.link_enc_cfg_ctx.link_enc_assignments[i].eng_id;
|
||||
stream = state->res_ctx.link_enc_cfg_ctx.link_enc_assignments[i].stream;
|
||||
if (eng_id != ENGINE_ID_UNKNOWN)
|
||||
state->res_ctx.link_enc_cfg_ctx.link_enc_avail[eng_id - ENGINE_ID_DIGA] = eng_id;
|
||||
if (stream)
|
||||
stream->link_enc = NULL;
|
||||
state->res_ctx.link_enc_cfg_ctx.link_enc_assignments[i].eng_id = ENGINE_ID_UNKNOWN;
|
||||
if (state->res_ctx.link_enc_cfg_ctx.link_enc_assignments[i].stream != NULL) {
|
||||
dc_stream_release(state->res_ctx.link_enc_cfg_ctx.link_enc_assignments[i].stream);
|
||||
state->res_ctx.link_enc_cfg_ctx.link_enc_assignments[i].stream = NULL;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void link_enc_cfg_init(
|
||||
struct dc *dc,
|
||||
struct dc_state *state)
|
||||
{
|
||||
int i;
|
||||
|
||||
for (i = 0; i < dc->res_pool->res_cap->num_dig_link_enc; i++) {
|
||||
if (dc->res_pool->link_encoders[i])
|
||||
@ -266,8 +260,13 @@ void link_enc_cfg_init(
|
||||
else
|
||||
state->res_ctx.link_enc_cfg_ctx.link_enc_avail[i] = ENGINE_ID_UNKNOWN;
|
||||
}
|
||||
}
|
||||
|
||||
clear_enc_assignments(state);
|
||||
void link_enc_cfg_init(
|
||||
const struct dc *dc,
|
||||
struct dc_state *state)
|
||||
{
|
||||
clear_enc_assignments(dc, state);
|
||||
|
||||
state->res_ctx.link_enc_cfg_ctx.mode = LINK_ENC_CFG_STEADY;
|
||||
}
|
||||
@ -284,12 +283,9 @@ void link_enc_cfg_link_encs_assign(
|
||||
|
||||
ASSERT(state->stream_count == stream_count);
|
||||
|
||||
if (stream_count == 0)
|
||||
clear_enc_assignments(state);
|
||||
|
||||
/* Release DIG link encoder resources before running assignment algorithm. */
|
||||
for (i = 0; i < stream_count; i++)
|
||||
dc->res_pool->funcs->link_enc_unassign(state, streams[i]);
|
||||
for (i = 0; i < dc->current_state->stream_count; i++)
|
||||
dc->res_pool->funcs->link_enc_unassign(state, dc->current_state->streams[i]);
|
||||
|
||||
for (i = 0; i < MAX_PIPES; i++)
|
||||
ASSERT(state->res_ctx.link_enc_cfg_ctx.link_enc_assignments[i].valid == false);
|
||||
@ -544,6 +540,7 @@ bool link_enc_cfg_validate(struct dc *dc, struct dc_state *state)
|
||||
uint8_t dig_stream_count = 0;
|
||||
int matching_stream_ptrs = 0;
|
||||
int eng_ids_per_ep_id[MAX_PIPES] = {0};
|
||||
int valid_bitmap = 0;
|
||||
|
||||
/* (1) No. valid entries same as stream count. */
|
||||
for (i = 0; i < MAX_PIPES; i++) {
|
||||
@ -625,5 +622,15 @@ bool link_enc_cfg_validate(struct dc *dc, struct dc_state *state)
|
||||
is_valid = valid_entries && valid_stream_ptrs && valid_uniqueness && valid_avail && valid_streams;
|
||||
ASSERT(is_valid);
|
||||
|
||||
if (is_valid == false) {
|
||||
valid_bitmap =
|
||||
(valid_entries & 0x1) |
|
||||
((valid_stream_ptrs & 0x1) << 1) |
|
||||
((valid_uniqueness & 0x1) << 2) |
|
||||
((valid_avail & 0x1) << 3) |
|
||||
((valid_streams & 0x1) << 4);
|
||||
dm_error("Invalid link encoder assignments: 0x%x\n", valid_bitmap);
|
||||
}
|
||||
|
||||
return is_valid;
|
||||
}
|
||||
|
||||
@ -2078,7 +2078,6 @@ static void mark_seamless_boot_stream(
|
||||
{
|
||||
struct dc_bios *dcb = dc->ctx->dc_bios;
|
||||
|
||||
/* TODO: Check Linux */
|
||||
if (dc->config.allow_seamless_boot_optimization &&
|
||||
!dcb->funcs->is_accelerated_mode(dcb)) {
|
||||
if (dc_validate_seamless_boot_timing(dc, stream->sink, &stream->timing))
|
||||
@ -2224,6 +2223,9 @@ void dc_resource_state_construct(
|
||||
struct dc_state *dst_ctx)
|
||||
{
|
||||
dst_ctx->clk_mgr = dc->clk_mgr;
|
||||
|
||||
/* Initialise DIG link encoder resource tracking variables. */
|
||||
link_enc_cfg_init(dc, dst_ctx);
|
||||
}
|
||||
|
||||
|
||||
@ -2506,17 +2508,7 @@ static void set_avi_info_frame(
|
||||
|
||||
/* TODO : We should handle YCC quantization */
|
||||
/* but we do not have matrix calculation */
|
||||
if (stream->qy_bit == 1) {
|
||||
if (color_space == COLOR_SPACE_SRGB ||
|
||||
color_space == COLOR_SPACE_2020_RGB_FULLRANGE)
|
||||
hdmi_info.bits.YQ0_YQ1 = YYC_QUANTIZATION_LIMITED_RANGE;
|
||||
else if (color_space == COLOR_SPACE_SRGB_LIMITED ||
|
||||
color_space == COLOR_SPACE_2020_RGB_LIMITEDRANGE)
|
||||
hdmi_info.bits.YQ0_YQ1 = YYC_QUANTIZATION_LIMITED_RANGE;
|
||||
else
|
||||
hdmi_info.bits.YQ0_YQ1 = YYC_QUANTIZATION_LIMITED_RANGE;
|
||||
} else
|
||||
hdmi_info.bits.YQ0_YQ1 = YYC_QUANTIZATION_LIMITED_RANGE;
|
||||
hdmi_info.bits.YQ0_YQ1 = YYC_QUANTIZATION_LIMITED_RANGE;
|
||||
|
||||
///VIC
|
||||
format = stream->timing.timing_3d_format;
|
||||
|
||||
@ -33,14 +33,6 @@
|
||||
* Private functions
|
||||
******************************************************************************/
|
||||
|
||||
static void dc_sink_destruct(struct dc_sink *sink)
|
||||
{
|
||||
if (sink->dc_container_id) {
|
||||
kfree(sink->dc_container_id);
|
||||
sink->dc_container_id = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
static bool dc_sink_construct(struct dc_sink *sink, const struct dc_sink_init_data *init_params)
|
||||
{
|
||||
|
||||
@ -75,7 +67,7 @@ void dc_sink_retain(struct dc_sink *sink)
|
||||
static void dc_sink_free(struct kref *kref)
|
||||
{
|
||||
struct dc_sink *sink = container_of(kref, struct dc_sink, refcount);
|
||||
dc_sink_destruct(sink);
|
||||
kfree(sink->dc_container_id);
|
||||
kfree(sink);
|
||||
}
|
||||
|
||||
|
||||
@ -47,7 +47,7 @@ struct aux_payload;
|
||||
struct set_config_cmd_payload;
|
||||
struct dmub_notification;
|
||||
|
||||
#define DC_VER "3.2.160"
|
||||
#define DC_VER "3.2.163"
|
||||
|
||||
#define MAX_SURFACES 3
|
||||
#define MAX_PLANES 6
|
||||
@ -188,6 +188,7 @@ struct dc_caps {
|
||||
#if defined(CONFIG_DRM_AMD_DC_DCN)
|
||||
bool dp_hpo;
|
||||
#endif
|
||||
bool edp_dsc_support;
|
||||
bool vbios_lttpr_aware;
|
||||
bool vbios_lttpr_enable;
|
||||
};
|
||||
@ -573,6 +574,8 @@ struct dc_debug_options {
|
||||
bool native422_support;
|
||||
bool disable_dsc;
|
||||
enum visual_confirm visual_confirm;
|
||||
int visual_confirm_rect_height;
|
||||
|
||||
bool sanity_checks;
|
||||
bool max_disp_clk;
|
||||
bool surface_trace;
|
||||
@ -667,6 +670,8 @@ struct dc_debug_options {
|
||||
bool validate_dml_output;
|
||||
bool enable_dmcub_surface_flip;
|
||||
bool usbc_combo_phy_reset_wa;
|
||||
bool disable_dsc_edp;
|
||||
unsigned int force_dsc_edp_policy;
|
||||
bool enable_dram_clock_change_one_display_vactive;
|
||||
#if defined(CONFIG_DRM_AMD_DC_DCN)
|
||||
/* TODO - remove once tested */
|
||||
|
||||
@ -115,13 +115,44 @@ void dc_dmub_srv_wait_idle(struct dc_dmub_srv *dc_dmub_srv)
|
||||
}
|
||||
}
|
||||
|
||||
void dc_dmub_srv_clear_inbox0_ack(struct dc_dmub_srv *dmub_srv)
|
||||
{
|
||||
struct dmub_srv *dmub = dmub_srv->dmub;
|
||||
struct dc_context *dc_ctx = dmub_srv->ctx;
|
||||
enum dmub_status status = DMUB_STATUS_OK;
|
||||
|
||||
status = dmub_srv_clear_inbox0_ack(dmub);
|
||||
if (status != DMUB_STATUS_OK) {
|
||||
DC_ERROR("Error clearing INBOX0 ack: status=%d\n", status);
|
||||
dc_dmub_srv_log_diagnostic_data(dmub_srv);
|
||||
}
|
||||
}
|
||||
|
||||
void dc_dmub_srv_wait_for_inbox0_ack(struct dc_dmub_srv *dmub_srv)
|
||||
{
|
||||
struct dmub_srv *dmub = dmub_srv->dmub;
|
||||
struct dc_context *dc_ctx = dmub_srv->ctx;
|
||||
enum dmub_status status = DMUB_STATUS_OK;
|
||||
|
||||
status = dmub_srv_wait_for_inbox0_ack(dmub, 100000);
|
||||
if (status != DMUB_STATUS_OK) {
|
||||
DC_ERROR("Error waiting for INBOX0 HW Lock Ack\n");
|
||||
dc_dmub_srv_log_diagnostic_data(dmub_srv);
|
||||
}
|
||||
}
|
||||
|
||||
void dc_dmub_srv_send_inbox0_cmd(struct dc_dmub_srv *dmub_srv,
|
||||
union dmub_inbox0_data_register data)
|
||||
{
|
||||
struct dmub_srv *dmub = dmub_srv->dmub;
|
||||
if (dmub->hw_funcs.send_inbox0_cmd)
|
||||
dmub->hw_funcs.send_inbox0_cmd(dmub, data);
|
||||
// TODO: Add wait command -- poll register for ACK
|
||||
struct dc_context *dc_ctx = dmub_srv->ctx;
|
||||
enum dmub_status status = DMUB_STATUS_OK;
|
||||
|
||||
status = dmub_srv_send_inbox0_cmd(dmub, data);
|
||||
if (status != DMUB_STATUS_OK) {
|
||||
DC_ERROR("Error sending INBOX0 cmd\n");
|
||||
dc_dmub_srv_log_diagnostic_data(dmub_srv);
|
||||
}
|
||||
}
|
||||
|
||||
bool dc_dmub_srv_cmd_with_reply_data(struct dc_dmub_srv *dc_dmub_srv, union dmub_rb_cmd *cmd)
|
||||
|
||||
@ -68,6 +68,8 @@ bool dc_dmub_srv_get_dmub_outbox0_msg(const struct dc *dc, struct dmcub_trace_bu
|
||||
|
||||
void dc_dmub_trace_event_control(struct dc *dc, bool enable);
|
||||
|
||||
void dc_dmub_srv_clear_inbox0_ack(struct dc_dmub_srv *dmub_srv);
|
||||
void dc_dmub_srv_wait_for_inbox0_ack(struct dc_dmub_srv *dmub_srv);
|
||||
void dc_dmub_srv_send_inbox0_cmd(struct dc_dmub_srv *dmub_srv, union dmub_inbox0_data_register data);
|
||||
|
||||
bool dc_dmub_srv_get_diagnostic_data(struct dc_dmub_srv *dc_dmub_srv, struct dmub_diagnostic_data *dmub_oca);
|
||||
|
||||
@ -113,6 +113,7 @@ struct dc_link {
|
||||
* DIG encoder. */
|
||||
bool is_dig_mapping_flexible;
|
||||
bool hpd_status; /* HPD status of link without physical HPD pin. */
|
||||
bool is_hpd_pending; /* Indicates a new received hpd */
|
||||
|
||||
bool edp_sink_present;
|
||||
|
||||
@ -287,6 +288,10 @@ bool dc_link_setup_psr(struct dc_link *dc_link,
|
||||
|
||||
void dc_link_get_psr_residency(const struct dc_link *link, uint32_t *residency);
|
||||
|
||||
void dc_link_blank_all_dp_displays(struct dc *dc);
|
||||
|
||||
void dc_link_blank_dp_stream(struct dc_link *link, bool hw_init);
|
||||
|
||||
/* Request DC to detect if there is a Panel connected.
|
||||
* boot - If this call is during initial boot.
|
||||
* Return false for any type of detection failure or MST detection
|
||||
@ -298,7 +303,7 @@ enum dc_detect_reason {
|
||||
DETECT_REASON_HPD,
|
||||
DETECT_REASON_HPDRX,
|
||||
DETECT_REASON_FALLBACK,
|
||||
DETECT_REASON_RETRAIN
|
||||
DETECT_REASON_RETRAIN,
|
||||
};
|
||||
|
||||
bool dc_link_detect(struct dc_link *dc_link, enum dc_detect_reason reason);
|
||||
|
||||
@ -67,9 +67,6 @@ static void write_indirect_azalia_reg(struct audio *audio,
|
||||
/* AZALIA_F0_CODEC_ENDPOINT_DATA endpoint data */
|
||||
REG_SET(AZALIA_F0_CODEC_ENDPOINT_DATA, 0,
|
||||
AZALIA_ENDPOINT_REG_DATA, reg_data);
|
||||
|
||||
DC_LOG_HW_AUDIO("AUDIO:write_indirect_azalia_reg: index: %u data: %u\n",
|
||||
reg_index, reg_data);
|
||||
}
|
||||
|
||||
static uint32_t read_indirect_azalia_reg(struct audio *audio, uint32_t reg_index)
|
||||
@ -85,9 +82,6 @@ static uint32_t read_indirect_azalia_reg(struct audio *audio, uint32_t reg_index
|
||||
/* AZALIA_F0_CODEC_ENDPOINT_DATA endpoint data */
|
||||
value = REG_READ(AZALIA_F0_CODEC_ENDPOINT_DATA);
|
||||
|
||||
DC_LOG_HW_AUDIO("AUDIO:read_indirect_azalia_reg: index: %u data: %u\n",
|
||||
reg_index, value);
|
||||
|
||||
return value;
|
||||
}
|
||||
|
||||
|
||||
@ -113,6 +113,7 @@ struct dce_audio_shift {
|
||||
uint8_t DCCG_AUDIO_DTO2_USE_512FBR_DTO;
|
||||
uint32_t DCCG_AUDIO_DTO0_USE_512FBR_DTO;
|
||||
uint32_t DCCG_AUDIO_DTO1_USE_512FBR_DTO;
|
||||
uint32_t CLOCK_GATING_DISABLE;
|
||||
};
|
||||
|
||||
struct dce_audio_mask {
|
||||
@ -132,6 +133,7 @@ struct dce_audio_mask {
|
||||
uint32_t DCCG_AUDIO_DTO2_USE_512FBR_DTO;
|
||||
uint32_t DCCG_AUDIO_DTO0_USE_512FBR_DTO;
|
||||
uint32_t DCCG_AUDIO_DTO1_USE_512FBR_DTO;
|
||||
uint32_t CLOCK_GATING_DISABLE;
|
||||
|
||||
};
|
||||
|
||||
|
||||
@ -56,8 +56,11 @@ void dmub_hw_lock_mgr_inbox0_cmd(struct dc_dmub_srv *dmub_srv,
|
||||
union dmub_inbox0_cmd_lock_hw hw_lock_cmd)
|
||||
{
|
||||
union dmub_inbox0_data_register data = { 0 };
|
||||
|
||||
data.inbox0_cmd_lock_hw = hw_lock_cmd;
|
||||
dc_dmub_srv_clear_inbox0_ack(dmub_srv);
|
||||
dc_dmub_srv_send_inbox0_cmd(dmub_srv, data);
|
||||
dc_dmub_srv_wait_for_inbox0_ack(dmub_srv);
|
||||
}
|
||||
|
||||
bool should_use_dmub_lock(struct dc_link *link)
|
||||
|
||||
@ -230,7 +230,7 @@ static void dmub_psr_set_level(struct dmub_psr *dmub, uint16_t psr_level, uint8_
|
||||
/**
|
||||
* Set PSR power optimization flags.
|
||||
*/
|
||||
static void dmub_psr_set_power_opt(struct dmub_psr *dmub, unsigned int power_opt)
|
||||
static void dmub_psr_set_power_opt(struct dmub_psr *dmub, unsigned int power_opt, uint8_t panel_inst)
|
||||
{
|
||||
union dmub_rb_cmd cmd;
|
||||
struct dc_context *dc = dmub->ctx;
|
||||
@ -239,7 +239,9 @@ static void dmub_psr_set_power_opt(struct dmub_psr *dmub, unsigned int power_opt
|
||||
cmd.psr_set_power_opt.header.type = DMUB_CMD__PSR;
|
||||
cmd.psr_set_power_opt.header.sub_type = DMUB_CMD__SET_PSR_POWER_OPT;
|
||||
cmd.psr_set_power_opt.header.payload_bytes = sizeof(struct dmub_cmd_psr_set_power_opt_data);
|
||||
cmd.psr_set_power_opt.psr_set_power_opt_data.cmd_version = DMUB_CMD_PSR_CONTROL_VERSION_1;
|
||||
cmd.psr_set_power_opt.psr_set_power_opt_data.power_opt = power_opt;
|
||||
cmd.psr_set_power_opt.psr_set_power_opt_data.panel_inst = panel_inst;
|
||||
|
||||
dc_dmub_srv_cmd_queue(dc->dmub_srv, &cmd);
|
||||
dc_dmub_srv_cmd_execute(dc->dmub_srv);
|
||||
@ -327,6 +329,16 @@ static bool dmub_psr_copy_settings(struct dmub_psr *dmub,
|
||||
copy_settings_data->fec_enable_delay_in100us = link->dc->debug.fec_enable_delay_in100us;
|
||||
copy_settings_data->cmd_version = DMUB_CMD_PSR_CONTROL_VERSION_1;
|
||||
copy_settings_data->panel_inst = panel_inst;
|
||||
copy_settings_data->dsc_enable_status = (pipe_ctx->stream->timing.flags.DSC == 1);
|
||||
|
||||
if (link->fec_state == dc_link_fec_enabled &&
|
||||
(!memcmp(link->dpcd_caps.sink_dev_id_str, DP_SINK_DEVICE_STR_ID_1,
|
||||
sizeof(link->dpcd_caps.sink_dev_id_str)) ||
|
||||
!memcmp(link->dpcd_caps.sink_dev_id_str, DP_SINK_DEVICE_STR_ID_2,
|
||||
sizeof(link->dpcd_caps.sink_dev_id_str))))
|
||||
copy_settings_data->debug.bitfields.force_wakeup_by_tps3 = 1;
|
||||
else
|
||||
copy_settings_data->debug.bitfields.force_wakeup_by_tps3 = 0;
|
||||
|
||||
dc_dmub_srv_cmd_queue(dc->dmub_srv, &cmd);
|
||||
dc_dmub_srv_cmd_execute(dc->dmub_srv);
|
||||
|
||||
@ -46,7 +46,7 @@ struct dmub_psr_funcs {
|
||||
void (*psr_force_static)(struct dmub_psr *dmub, uint8_t panel_inst);
|
||||
void (*psr_get_residency)(struct dmub_psr *dmub, uint32_t *residency,
|
||||
uint8_t panel_inst);
|
||||
void (*psr_set_power_opt)(struct dmub_psr *dmub, unsigned int power_opt);
|
||||
void (*psr_set_power_opt)(struct dmub_psr *dmub, unsigned int power_opt, uint8_t panel_inst);
|
||||
};
|
||||
|
||||
struct dmub_psr *dmub_psr_create(struct dc_context *ctx);
|
||||
|
||||
@ -1602,6 +1602,11 @@ static enum dc_status apply_single_controller_ctx_to_hw(
|
||||
pipe_ctx->stream_res.stream_enc,
|
||||
pipe_ctx->stream_res.tg->inst);
|
||||
|
||||
if (dc_is_dp_signal(pipe_ctx->stream->signal) &&
|
||||
pipe_ctx->stream_res.stream_enc->funcs->reset_fifo)
|
||||
pipe_ctx->stream_res.stream_enc->funcs->reset_fifo(
|
||||
pipe_ctx->stream_res.stream_enc);
|
||||
|
||||
if (dc_is_dp_signal(pipe_ctx->stream->signal))
|
||||
dp_source_sequence_trace(link, DPCD_SOURCE_SEQ_AFTER_CONNECT_DIG_FE_OTG);
|
||||
|
||||
@ -1655,30 +1660,12 @@ static enum dc_status apply_single_controller_ctx_to_hw(
|
||||
|
||||
static void power_down_encoders(struct dc *dc)
|
||||
{
|
||||
int i, j;
|
||||
int i;
|
||||
|
||||
for (i = 0; i < dc->link_count; i++) {
|
||||
enum signal_type signal = dc->links[i]->connector_signal;
|
||||
|
||||
if ((signal == SIGNAL_TYPE_EDP) ||
|
||||
(signal == SIGNAL_TYPE_DISPLAY_PORT)) {
|
||||
if (dc->links[i]->link_enc->funcs->get_dig_frontend &&
|
||||
dc->links[i]->link_enc->funcs->is_dig_enabled(dc->links[i]->link_enc)) {
|
||||
unsigned int fe = dc->links[i]->link_enc->funcs->get_dig_frontend(
|
||||
dc->links[i]->link_enc);
|
||||
|
||||
for (j = 0; j < dc->res_pool->stream_enc_count; j++) {
|
||||
if (fe == dc->res_pool->stream_enc[j]->id) {
|
||||
dc->res_pool->stream_enc[j]->funcs->dp_blank(dc->links[i],
|
||||
dc->res_pool->stream_enc[j]);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!dc->links[i]->wa_flags.dp_keep_receiver_powered)
|
||||
dp_receiver_power_ctrl(dc->links[i], false);
|
||||
}
|
||||
dc_link_blank_dp_stream(dc->links[i], false);
|
||||
|
||||
if (signal != SIGNAL_TYPE_EDP)
|
||||
signal = SIGNAL_TYPE_NONE;
|
||||
@ -1846,7 +1833,7 @@ void dce110_enable_accelerated_mode(struct dc *dc, struct dc_state *context)
|
||||
}
|
||||
}
|
||||
// We are trying to enable eDP, don't power down VDD
|
||||
if (edp_stream_num)
|
||||
if (edp_stream_num && can_apply_edp_fast_boot)
|
||||
keep_edp_vdd_on = true;
|
||||
}
|
||||
|
||||
|
||||
@ -39,6 +39,10 @@
|
||||
#define BLACK_OFFSET_RGB_Y 0x0
|
||||
#define BLACK_OFFSET_CBCR 0x8000
|
||||
|
||||
#define VISUAL_CONFIRM_RECT_HEIGHT_DEFAULT 3
|
||||
#define VISUAL_CONFIRM_RECT_HEIGHT_MIN 1
|
||||
#define VISUAL_CONFIRM_RECT_HEIGHT_MAX 10
|
||||
|
||||
#define REG(reg)\
|
||||
dpp->tf_regs->reg
|
||||
|
||||
@ -685,9 +689,17 @@ static void dpp1_dscl_set_recout(struct dcn10_dpp *dpp,
|
||||
const struct rect *recout)
|
||||
{
|
||||
int visual_confirm_on = 0;
|
||||
unsigned short visual_confirm_rect_height = VISUAL_CONFIRM_RECT_HEIGHT_DEFAULT;
|
||||
|
||||
if (dpp->base.ctx->dc->debug.visual_confirm != VISUAL_CONFIRM_DISABLE)
|
||||
visual_confirm_on = 1;
|
||||
|
||||
/* Check bounds to ensure the VC bar height was set to a sane value */
|
||||
if ((dpp->base.ctx->dc->debug.visual_confirm_rect_height >= VISUAL_CONFIRM_RECT_HEIGHT_MIN) &&
|
||||
(dpp->base.ctx->dc->debug.visual_confirm_rect_height <= VISUAL_CONFIRM_RECT_HEIGHT_MAX)) {
|
||||
visual_confirm_rect_height = dpp->base.ctx->dc->debug.visual_confirm_rect_height;
|
||||
}
|
||||
|
||||
REG_SET_2(RECOUT_START, 0,
|
||||
/* First pixel of RECOUT in the active OTG area */
|
||||
RECOUT_START_X, recout->x,
|
||||
@ -699,7 +711,7 @@ static void dpp1_dscl_set_recout(struct dcn10_dpp *dpp,
|
||||
RECOUT_WIDTH, recout->width,
|
||||
/* Number of RECOUT vertical lines */
|
||||
RECOUT_HEIGHT, recout->height
|
||||
- visual_confirm_on * 2 * (dpp->base.inst + 1));
|
||||
- visual_confirm_on * 2 * (dpp->base.inst + visual_confirm_rect_height));
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@ -1362,11 +1362,48 @@ void dcn10_init_pipes(struct dc *dc, struct dc_state *context)
|
||||
|
||||
tg->funcs->tg_init(tg);
|
||||
}
|
||||
|
||||
/* Power gate DSCs */
|
||||
if (hws->funcs.dsc_pg_control != NULL) {
|
||||
uint32_t num_opps = 0;
|
||||
uint32_t opp_id_src0 = OPP_ID_INVALID;
|
||||
uint32_t opp_id_src1 = OPP_ID_INVALID;
|
||||
|
||||
// Step 1: To find out which OPTC is running & OPTC DSC is ON
|
||||
for (i = 0; i < dc->res_pool->res_cap->num_timing_generator; i++) {
|
||||
uint32_t optc_dsc_state = 0;
|
||||
struct timing_generator *tg = dc->res_pool->timing_generators[i];
|
||||
|
||||
if (tg->funcs->is_tg_enabled(tg)) {
|
||||
if (tg->funcs->get_dsc_status)
|
||||
tg->funcs->get_dsc_status(tg, &optc_dsc_state);
|
||||
// Only one OPTC with DSC is ON, so if we got one result, we would exit this block.
|
||||
// non-zero value is DSC enabled
|
||||
if (optc_dsc_state != 0) {
|
||||
tg->funcs->get_optc_source(tg, &num_opps, &opp_id_src0, &opp_id_src1);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Step 2: To power down DSC but skip DSC of running OPTC
|
||||
for (i = 0; i < dc->res_pool->res_cap->num_dsc; i++) {
|
||||
struct dcn_dsc_state s = {0};
|
||||
|
||||
dc->res_pool->dscs[i]->funcs->dsc_read_state(dc->res_pool->dscs[i], &s);
|
||||
|
||||
if ((s.dsc_opp_source == opp_id_src0 || s.dsc_opp_source == opp_id_src1) &&
|
||||
s.dsc_clock_en && s.dsc_fw_en)
|
||||
continue;
|
||||
|
||||
hws->funcs.dsc_pg_control(hws, dc->res_pool->dscs[i]->inst, false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void dcn10_init_hw(struct dc *dc)
|
||||
{
|
||||
int i, j;
|
||||
int i;
|
||||
struct abm *abm = dc->res_pool->abm;
|
||||
struct dmcu *dmcu = dc->res_pool->dmcu;
|
||||
struct dce_hwseq *hws = dc->hwseq;
|
||||
@ -1468,43 +1505,8 @@ void dcn10_init_hw(struct dc *dc)
|
||||
dmub_enable_outbox_notification(dc);
|
||||
|
||||
/* we want to turn off all dp displays before doing detection */
|
||||
if (dc->config.power_down_display_on_boot) {
|
||||
uint8_t dpcd_power_state = '\0';
|
||||
enum dc_status status = DC_ERROR_UNEXPECTED;
|
||||
|
||||
for (i = 0; i < dc->link_count; i++) {
|
||||
if (dc->links[i]->connector_signal != SIGNAL_TYPE_DISPLAY_PORT)
|
||||
continue;
|
||||
|
||||
/* DP 2.0 requires that LTTPR Caps be read first */
|
||||
dp_retrieve_lttpr_cap(dc->links[i]);
|
||||
|
||||
/*
|
||||
* If any of the displays are lit up turn them off.
|
||||
* The reason is that some MST hubs cannot be turned off
|
||||
* completely until we tell them to do so.
|
||||
* If not turned off, then displays connected to MST hub
|
||||
* won't light up.
|
||||
*/
|
||||
status = core_link_read_dpcd(dc->links[i], DP_SET_POWER,
|
||||
&dpcd_power_state, sizeof(dpcd_power_state));
|
||||
if (status == DC_OK && dpcd_power_state == DP_POWER_STATE_D0) {
|
||||
/* blank dp stream before power off receiver*/
|
||||
if (dc->links[i]->link_enc->funcs->get_dig_frontend) {
|
||||
unsigned int fe = dc->links[i]->link_enc->funcs->get_dig_frontend(dc->links[i]->link_enc);
|
||||
|
||||
for (j = 0; j < dc->res_pool->stream_enc_count; j++) {
|
||||
if (fe == dc->res_pool->stream_enc[j]->id) {
|
||||
dc->res_pool->stream_enc[j]->funcs->dp_blank(dc->links[i],
|
||||
dc->res_pool->stream_enc[j]);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
dp_receiver_power_ctrl(dc->links[i], false);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (dc->config.power_down_display_on_boot)
|
||||
dc_link_blank_all_dp_displays(dc);
|
||||
|
||||
/* If taking control over from VBIOS, we may want to optimize our first
|
||||
* mode set, so we need to skip powering down pipes until we know which
|
||||
@ -1637,7 +1639,7 @@ void dcn10_reset_hw_ctx_wrap(
|
||||
|
||||
dcn10_reset_back_end_for_pipe(dc, pipe_ctx_old, dc->current_state);
|
||||
if (hws->funcs.enable_stream_gating)
|
||||
hws->funcs.enable_stream_gating(dc, pipe_ctx);
|
||||
hws->funcs.enable_stream_gating(dc, pipe_ctx_old);
|
||||
if (old_clk)
|
||||
old_clk->funcs->cs_power_down(old_clk);
|
||||
}
|
||||
@ -2624,7 +2626,7 @@ static void dcn10_update_dchubp_dpp(
|
||||
/* new calculated dispclk, dppclk are stored in
|
||||
* context->bw_ctx.bw.dcn.clk.dispclk_khz / dppclk_khz. current
|
||||
* dispclk, dppclk are from dc->clk_mgr->clks.dispclk_khz.
|
||||
* dcn_validate_bandwidth compute new dispclk, dppclk.
|
||||
* dcn10_validate_bandwidth compute new dispclk, dppclk.
|
||||
* dispclk will put in use after optimize_bandwidth when
|
||||
* ramp_up_dispclk_with_dpp is called.
|
||||
* there are two places for dppclk be put in use. One location
|
||||
@ -2638,7 +2640,7 @@ static void dcn10_update_dchubp_dpp(
|
||||
* for example, eDP + external dp, change resolution of DP from
|
||||
* 1920x1080x144hz to 1280x960x60hz.
|
||||
* before change: dispclk = 337889 dppclk = 337889
|
||||
* change mode, dcn_validate_bandwidth calculate
|
||||
* change mode, dcn10_validate_bandwidth calculate
|
||||
* dispclk = 143122 dppclk = 143122
|
||||
* update_dchubp_dpp be executed before dispclk be updated,
|
||||
* dispclk = 337889, but dppclk use new value dispclk /2 =
|
||||
|
||||
@ -978,10 +978,8 @@ static void dcn10_resource_destruct(struct dcn10_resource_pool *pool)
|
||||
pool->base.mpc = NULL;
|
||||
}
|
||||
|
||||
if (pool->base.hubbub != NULL) {
|
||||
kfree(pool->base.hubbub);
|
||||
pool->base.hubbub = NULL;
|
||||
}
|
||||
kfree(pool->base.hubbub);
|
||||
pool->base.hubbub = NULL;
|
||||
|
||||
for (i = 0; i < pool->base.pipe_count; i++) {
|
||||
if (pool->base.opps[i] != NULL)
|
||||
@ -1011,14 +1009,10 @@ static void dcn10_resource_destruct(struct dcn10_resource_pool *pool)
|
||||
for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
|
||||
if (pool->base.engines[i] != NULL)
|
||||
dce110_engine_destroy(&pool->base.engines[i]);
|
||||
if (pool->base.hw_i2cs[i] != NULL) {
|
||||
kfree(pool->base.hw_i2cs[i]);
|
||||
pool->base.hw_i2cs[i] = NULL;
|
||||
}
|
||||
if (pool->base.sw_i2cs[i] != NULL) {
|
||||
kfree(pool->base.sw_i2cs[i]);
|
||||
pool->base.sw_i2cs[i] = NULL;
|
||||
}
|
||||
kfree(pool->base.hw_i2cs[i]);
|
||||
pool->base.hw_i2cs[i] = NULL;
|
||||
kfree(pool->base.sw_i2cs[i]);
|
||||
pool->base.sw_i2cs[i] = NULL;
|
||||
}
|
||||
|
||||
for (i = 0; i < pool->base.audio_count; i++) {
|
||||
@ -1320,7 +1314,7 @@ static const struct resource_funcs dcn10_res_pool_funcs = {
|
||||
.destroy = dcn10_destroy_resource_pool,
|
||||
.link_enc_create = dcn10_link_encoder_create,
|
||||
.panel_cntl_create = dcn10_panel_cntl_create,
|
||||
.validate_bandwidth = dcn_validate_bandwidth,
|
||||
.validate_bandwidth = dcn10_validate_bandwidth,
|
||||
.acquire_idle_pipe_for_layer = dcn10_acquire_idle_pipe_for_layer,
|
||||
.validate_plane = dcn10_validate_plane,
|
||||
.validate_global = dcn10_validate_global,
|
||||
|
||||
@ -902,6 +902,19 @@ void enc1_stream_encoder_stop_dp_info_packets(
|
||||
|
||||
}
|
||||
|
||||
void enc1_stream_encoder_reset_fifo(
|
||||
struct stream_encoder *enc)
|
||||
{
|
||||
struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc);
|
||||
|
||||
/* set DIG_START to 0x1 to reset FIFO */
|
||||
REG_UPDATE(DIG_FE_CNTL, DIG_START, 1);
|
||||
udelay(100);
|
||||
|
||||
/* write 0 to take the FIFO out of reset */
|
||||
REG_UPDATE(DIG_FE_CNTL, DIG_START, 0);
|
||||
}
|
||||
|
||||
void enc1_stream_encoder_dp_blank(
|
||||
struct dc_link *link,
|
||||
struct stream_encoder *enc)
|
||||
@ -1587,6 +1600,8 @@ static const struct stream_encoder_funcs dcn10_str_enc_funcs = {
|
||||
enc1_stream_encoder_send_immediate_sdp_message,
|
||||
.stop_dp_info_packets =
|
||||
enc1_stream_encoder_stop_dp_info_packets,
|
||||
.reset_fifo =
|
||||
enc1_stream_encoder_reset_fifo,
|
||||
.dp_blank =
|
||||
enc1_stream_encoder_dp_blank,
|
||||
.dp_unblank =
|
||||
|
||||
@ -626,6 +626,9 @@ void enc1_stream_encoder_send_immediate_sdp_message(
|
||||
void enc1_stream_encoder_stop_dp_info_packets(
|
||||
struct stream_encoder *enc);
|
||||
|
||||
void enc1_stream_encoder_reset_fifo(
|
||||
struct stream_encoder *enc);
|
||||
|
||||
void enc1_stream_encoder_dp_blank(
|
||||
struct dc_link *link,
|
||||
struct stream_encoder *enc);
|
||||
|
||||
@ -162,6 +162,8 @@ static void dsc2_read_state(struct display_stream_compressor *dsc, struct dcn_ds
|
||||
REG_GET(DSCC_PPS_CONFIG2, PIC_WIDTH, &s->dsc_pic_width);
|
||||
REG_GET(DSCC_PPS_CONFIG2, PIC_HEIGHT, &s->dsc_pic_height);
|
||||
REG_GET(DSCC_PPS_CONFIG7, SLICE_BPG_OFFSET, &s->dsc_slice_bpg_offset);
|
||||
REG_GET_2(DSCRM_DSC_FORWARD_CONFIG, DSCRM_DSC_FORWARD_EN, &s->dsc_fw_en,
|
||||
DSCRM_DSC_OPP_PIPE_SOURCE, &s->dsc_opp_source);
|
||||
}
|
||||
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
x
Reference in New Issue
Block a user