iv/linux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

habanalabs: add an option to delay a device reset

Browse Source 
Several H/W events can be sent adjacently, even due to a single error.
If a hard-reset is triggered as part of handling one of these events,
the following events won't be handled.
The debug info from these missed events is important, sometimes even
more important than the one that was handled.

To allow handling these close events, add an option to delay a device
reset and use it when resetting due to H/W events.

Signed-off-by: Tomer Tayar <ttayar@habana.ai>
Reviewed-by: Oded Gabbay <ogabbay@kernel.org>
Signed-off-by: Oded Gabbay <ogabbay@kernel.org>
This commit is contained in:
Tomer Tayar 2022-02-23 14:46:37 +02:00 committed by Oded Gabbay
parent 9c27896ac1
commit b0106bc6fe
3 changed files with 12 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
drivers/misc/habanalabs/common/device.c
View File

@ -13,6 +13,8 @@
#include <linux/pci.h>
#include <linux/hwmon.h>
#define HL_RESET_DELAY_USEC 10000 /* 10ms */
enum hl_device_status hl_device_status(struct hl_device *hdev)
{
enum hl_device_status status;
@ -980,7 +982,7 @@ int hl_device_reset(struct hl_device *hdev, u32 flags)
{
bool hard_reset, from_hard_reset_thread, fw_reset, hard_instead_soft = false,
reset_upon_device_release = false, schedule_hard_reset = false,
skip_wq_flush = false;
skip_wq_flush, delay_reset;
u64 idle_mask[HL_BUSY_ENGINES_MASK_EXT_SIZE] = {0};
struct hl_ctx *ctx;
int i, rc;
@ -994,6 +996,7 @@ int hl_device_reset(struct hl_device *hdev, u32 flags)
from_hard_reset_thread = !!(flags & HL_DRV_RESET_FROM_RESET_THR);
fw_reset = !!(flags & HL_DRV_RESET_BYPASS_REQ_TO_FW);
skip_wq_flush = !!(flags & HL_DRV_RESET_DEV_RELEASE);
delay_reset = !!(flags & HL_DRV_RESET_DELAY);
if (!hard_reset && !hdev->asic_prop.supports_soft_reset) {
hard_instead_soft = true;
@ -1043,6 +1046,9 @@ do_reset:
hdev->reset_info.in_reset = 1;
spin_unlock(&hdev->reset_info.lock);
if (delay_reset)
usleep_range(HL_RESET_DELAY_USEC, HL_RESET_DELAY_USEC << 1);
handle_reset_trigger(hdev, flags);
/* This still allows the completion of some KDMA ops */

4
drivers/misc/habanalabs/common/habanalabs.h
View File

@ -142,6 +142,9 @@ enum hl_mmu_page_table_location {
*
* - HL_DRV_RESET_FW_FATAL_ERR
* Set if reset is due to a fatal error from FW
*
* - HL_DRV_RESET_DELAY
* Set if a delay should be added before the reset
*/
#define HL_DRV_RESET_HARD (1 << 0)
@ -151,6 +154,7 @@ enum hl_mmu_page_table_location {
#define HL_DRV_RESET_DEV_RELEASE (1 << 4)
#define HL_DRV_RESET_BYPASS_REQ_TO_FW (1 << 5)
#define HL_DRV_RESET_FW_FATAL_ERR (1 << 6)
#define HL_DRV_RESET_DELAY (1 << 7)
#define HL_MAX_SOBS_PER_MONITOR 8

2
drivers/misc/habanalabs/gaudi/gaudi.c
View File

@ -8199,7 +8199,7 @@ reset_device:
| HL_DRV_RESET_BYPASS_REQ_TO_FW
| fw_fatal_err_flag);
else if (hdev->hard_reset_on_fw_events)
hl_device_reset(hdev, HL_DRV_RESET_HARD | fw_fatal_err_flag);
hl_device_reset(hdev, HL_DRV_RESET_HARD | HL_DRV_RESET_DELAY | fw_fatal_err_flag);
else
hl_fw_unmask_irq(hdev, event_type);
}