2020-05-27 16:21:28 +01:00
|
|
|
// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
|
|
|
|
|
/* Copyright(c) 2014 - 2020 Intel Corporation */
|
2014-06-05 13:42:39 -07:00
|
|
|
#include <linux/mutex.h>
|
|
|
|
|
#include <linux/list.h>
|
|
|
|
|
#include <linux/bitops.h>
|
|
|
|
|
#include <linux/delay.h>
|
|
|
|
|
#include "adf_accel_devices.h"
|
|
|
|
|
#include "adf_cfg.h"
|
|
|
|
|
#include "adf_common_drv.h"
|
2023-05-26 17:48:59 +01:00
|
|
|
#include "adf_dbgfs.h"
|
2023-06-30 19:03:57 +02:00
|
|
|
#include "adf_heartbeat.h"
|
2023-10-20 15:49:28 +02:00
|
|
|
#include "adf_rl.h"
|
2023-10-20 11:32:52 +01:00
|
|
|
#include "adf_sysfs_ras_counters.h"
|
crypto: qat - add support for device telemetry
Expose through debugfs device telemetry data for QAT GEN4 devices.
This allows to gather metrics about the performance and the utilization
of a device. In particular, statistics on (1) the utilization of the
PCIe channel, (2) address translation, when SVA is enabled and (3) the
internal engines for crypto and data compression.
If telemetry is supported by the firmware, the driver allocates a DMA
region and a circular buffer. When telemetry is enabled, through the
`control` attribute in debugfs, the driver sends to the firmware, via
the admin interface, the `TL_START` command. This triggers the device to
periodically gather telemetry data from hardware registers and write it
into the DMA memory region. The device writes into the shared region
every second.
The driver, every 500ms, snapshots the DMA shared region into the
circular buffer. This is then used to compute basic metric
(min/max/average) on each counter, every time the `device_data` attribute
is queried.
Telemetry counters are exposed through debugfs in the folder
/sys/kernel/debug/qat_<device>_<BDF>/telemetry.
For details, refer to debugfs-driver-qat_telemetry in Documentation/ABI.
This patch is based on earlier work done by Wojciech Ziemba.
Signed-off-by: Lucas Segarra Fernandez <lucas.segarra.fernandez@intel.com>
Reviewed-by: Giovanni Cabiddu <giovanni.cabiddu@intel.com>
Reviewed-by: Damian Muszynski <damian.muszynski@intel.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2023-12-22 11:35:07 +01:00
|
|
|
#include "adf_telemetry.h"
|
2014-06-05 13:42:39 -07:00
|
|
|
|
|
|
|
|
static LIST_HEAD(service_table);
|
|
|
|
|
static DEFINE_MUTEX(service_lock);
|
|
|
|
|
|
|
|
|
|
static void adf_service_add(struct service_hndl *service)
|
|
|
|
|
{
|
|
|
|
|
mutex_lock(&service_lock);
|
|
|
|
|
list_add(&service->list, &service_table);
|
|
|
|
|
mutex_unlock(&service_lock);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
int adf_service_register(struct service_hndl *service)
|
|
|
|
|
{
|
2016-12-22 15:01:02 +00:00
|
|
|
memset(service->init_status, 0, sizeof(service->init_status));
|
|
|
|
|
memset(service->start_status, 0, sizeof(service->start_status));
|
2014-06-05 13:42:39 -07:00
|
|
|
adf_service_add(service);
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static void adf_service_remove(struct service_hndl *service)
|
|
|
|
|
{
|
|
|
|
|
mutex_lock(&service_lock);
|
|
|
|
|
list_del(&service->list);
|
|
|
|
|
mutex_unlock(&service_lock);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
int adf_service_unregister(struct service_hndl *service)
|
|
|
|
|
{
|
2016-12-22 15:01:02 +00:00
|
|
|
int i;
|
|
|
|
|
|
|
|
|
|
for (i = 0; i < ARRAY_SIZE(service->init_status); i++) {
|
|
|
|
|
if (service->init_status[i] || service->start_status[i]) {
|
|
|
|
|
pr_err("QAT: Could not remove active service\n");
|
|
|
|
|
return -EFAULT;
|
|
|
|
|
}
|
2014-06-05 13:42:39 -07:00
|
|
|
}
|
|
|
|
|
adf_service_remove(service);
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
crypto: qat - fix device reset flow
When the device needs a reset, e.g. when an uncorrectable PCIe AER event
occurs, various services/data structures need to be cleaned up, the
hardware reset and the services/data structures initialized and started.
The code to perform the cleanup and initialization was not performed when
a device reset was done.
This patch moves some of the initialization code out of the .probe entry-
point into a separate function that is now called during probe as well as
after the hardware has been reset. Similarly, a new function is added for
first cleaning up these services/data structures prior to resetting. The
new functions are adf_dev_init() and adf_dev_shutdown(), respectively, for
which there are already prototypes but no actual functions just yet and are
now called when the device is reset and during probe/cleanup of the driver.
The down and up flows via ioctl calls has similarly been updated.
In addition, there are two other bugs in the reset flow - one in the logic
for determining whether to schedule a device reset upon receiving an
uncorrectable AER event which prevents the reset flow from being initiated,
and another with clearing the status bit indicating a device is configured
(when resetting the device the configuration remains across the reset so
the bit should not be cleared, otherwise, the necessary services will not
be re-started in adf_dev_start() after the reset - clear the bit only when
actually deleting the configuration).
Signed-off-by: Bruce Allan <bruce.w.allan@intel.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2015-01-09 11:54:58 -08:00
|
|
|
* adf_dev_init() - Init data structures and services for the given accel device
|
|
|
|
|
* @accel_dev: Pointer to acceleration device.
|
2014-06-05 13:42:39 -07:00
|
|
|
*
|
crypto: qat - fix device reset flow
When the device needs a reset, e.g. when an uncorrectable PCIe AER event
occurs, various services/data structures need to be cleaned up, the
hardware reset and the services/data structures initialized and started.
The code to perform the cleanup and initialization was not performed when
a device reset was done.
This patch moves some of the initialization code out of the .probe entry-
point into a separate function that is now called during probe as well as
after the hardware has been reset. Similarly, a new function is added for
first cleaning up these services/data structures prior to resetting. The
new functions are adf_dev_init() and adf_dev_shutdown(), respectively, for
which there are already prototypes but no actual functions just yet and are
now called when the device is reset and during probe/cleanup of the driver.
The down and up flows via ioctl calls has similarly been updated.
In addition, there are two other bugs in the reset flow - one in the logic
for determining whether to schedule a device reset upon receiving an
uncorrectable AER event which prevents the reset flow from being initiated,
and another with clearing the status bit indicating a device is configured
(when resetting the device the configuration remains across the reset so
the bit should not be cleared, otherwise, the necessary services will not
be re-started in adf_dev_start() after the reset - clear the bit only when
actually deleting the configuration).
Signed-off-by: Bruce Allan <bruce.w.allan@intel.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2015-01-09 11:54:58 -08:00
|
|
|
* Initialize the ring data structures and the admin comms and arbitration
|
|
|
|
|
* services.
|
2014-06-05 13:42:39 -07:00
|
|
|
*
|
2015-07-24 13:18:26 -07:00
|
|
|
* Return: 0 on success, error code otherwise.
|
2014-06-05 13:42:39 -07:00
|
|
|
*/
|
2023-02-27 15:55:45 -05:00
|
|
|
static int adf_dev_init(struct adf_accel_dev *accel_dev)
|
2014-06-05 13:42:39 -07:00
|
|
|
{
|
|
|
|
|
struct service_hndl *service;
|
|
|
|
|
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
|
2021-08-12 21:21:13 +01:00
|
|
|
int ret;
|
2014-06-05 13:42:39 -07:00
|
|
|
|
crypto: qat - fix device reset flow
When the device needs a reset, e.g. when an uncorrectable PCIe AER event
occurs, various services/data structures need to be cleaned up, the
hardware reset and the services/data structures initialized and started.
The code to perform the cleanup and initialization was not performed when
a device reset was done.
This patch moves some of the initialization code out of the .probe entry-
point into a separate function that is now called during probe as well as
after the hardware has been reset. Similarly, a new function is added for
first cleaning up these services/data structures prior to resetting. The
new functions are adf_dev_init() and adf_dev_shutdown(), respectively, for
which there are already prototypes but no actual functions just yet and are
now called when the device is reset and during probe/cleanup of the driver.
The down and up flows via ioctl calls has similarly been updated.
In addition, there are two other bugs in the reset flow - one in the logic
for determining whether to schedule a device reset upon receiving an
uncorrectable AER event which prevents the reset flow from being initiated,
and another with clearing the status bit indicating a device is configured
(when resetting the device the configuration remains across the reset so
the bit should not be cleared, otherwise, the necessary services will not
be re-started in adf_dev_start() after the reset - clear the bit only when
actually deleting the configuration).
Signed-off-by: Bruce Allan <bruce.w.allan@intel.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2015-01-09 11:54:58 -08:00
|
|
|
if (!hw_data) {
|
|
|
|
|
dev_err(&GET_DEV(accel_dev),
|
2015-03-19 16:03:44 -07:00
|
|
|
"Failed to init device - hw_data not set\n");
|
crypto: qat - fix device reset flow
When the device needs a reset, e.g. when an uncorrectable PCIe AER event
occurs, various services/data structures need to be cleaned up, the
hardware reset and the services/data structures initialized and started.
The code to perform the cleanup and initialization was not performed when
a device reset was done.
This patch moves some of the initialization code out of the .probe entry-
point into a separate function that is now called during probe as well as
after the hardware has been reset. Similarly, a new function is added for
first cleaning up these services/data structures prior to resetting. The
new functions are adf_dev_init() and adf_dev_shutdown(), respectively, for
which there are already prototypes but no actual functions just yet and are
now called when the device is reset and during probe/cleanup of the driver.
The down and up flows via ioctl calls has similarly been updated.
In addition, there are two other bugs in the reset flow - one in the logic
for determining whether to schedule a device reset upon receiving an
uncorrectable AER event which prevents the reset flow from being initiated,
and another with clearing the status bit indicating a device is configured
(when resetting the device the configuration remains across the reset so
the bit should not be cleared, otherwise, the necessary services will not
be re-started in adf_dev_start() after the reset - clear the bit only when
actually deleting the configuration).
Signed-off-by: Bruce Allan <bruce.w.allan@intel.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2015-01-09 11:54:58 -08:00
|
|
|
return -EFAULT;
|
|
|
|
|
}
|
|
|
|
|
|
2021-12-16 09:13:30 +00:00
|
|
|
if (!test_bit(ADF_STATUS_CONFIGURED, &accel_dev->status) &&
|
|
|
|
|
!accel_dev->is_vf) {
|
2015-03-19 16:03:44 -07:00
|
|
|
dev_err(&GET_DEV(accel_dev), "Device not configured\n");
|
2014-06-05 13:42:39 -07:00
|
|
|
return -EFAULT;
|
|
|
|
|
}
|
crypto: qat - fix device reset flow
When the device needs a reset, e.g. when an uncorrectable PCIe AER event
occurs, various services/data structures need to be cleaned up, the
hardware reset and the services/data structures initialized and started.
The code to perform the cleanup and initialization was not performed when
a device reset was done.
This patch moves some of the initialization code out of the .probe entry-
point into a separate function that is now called during probe as well as
after the hardware has been reset. Similarly, a new function is added for
first cleaning up these services/data structures prior to resetting. The
new functions are adf_dev_init() and adf_dev_shutdown(), respectively, for
which there are already prototypes but no actual functions just yet and are
now called when the device is reset and during probe/cleanup of the driver.
The down and up flows via ioctl calls has similarly been updated.
In addition, there are two other bugs in the reset flow - one in the logic
for determining whether to schedule a device reset upon receiving an
uncorrectable AER event which prevents the reset flow from being initiated,
and another with clearing the status bit indicating a device is configured
(when resetting the device the configuration remains across the reset so
the bit should not be cleared, otherwise, the necessary services will not
be re-started in adf_dev_start() after the reset - clear the bit only when
actually deleting the configuration).
Signed-off-by: Bruce Allan <bruce.w.allan@intel.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2015-01-09 11:54:58 -08:00
|
|
|
|
|
|
|
|
if (adf_init_etr_data(accel_dev)) {
|
|
|
|
|
dev_err(&GET_DEV(accel_dev), "Failed initialize etr\n");
|
|
|
|
|
return -EFAULT;
|
|
|
|
|
}
|
|
|
|
|
|
2021-09-16 15:45:41 +01:00
|
|
|
if (hw_data->init_device && hw_data->init_device(accel_dev)) {
|
|
|
|
|
dev_err(&GET_DEV(accel_dev), "Failed to initialize device\n");
|
|
|
|
|
return -EFAULT;
|
|
|
|
|
}
|
|
|
|
|
|
crypto: qat - fix device reset flow
When the device needs a reset, e.g. when an uncorrectable PCIe AER event
occurs, various services/data structures need to be cleaned up, the
hardware reset and the services/data structures initialized and started.
The code to perform the cleanup and initialization was not performed when
a device reset was done.
This patch moves some of the initialization code out of the .probe entry-
point into a separate function that is now called during probe as well as
after the hardware has been reset. Similarly, a new function is added for
first cleaning up these services/data structures prior to resetting. The
new functions are adf_dev_init() and adf_dev_shutdown(), respectively, for
which there are already prototypes but no actual functions just yet and are
now called when the device is reset and during probe/cleanup of the driver.
The down and up flows via ioctl calls has similarly been updated.
In addition, there are two other bugs in the reset flow - one in the logic
for determining whether to schedule a device reset upon receiving an
uncorrectable AER event which prevents the reset flow from being initiated,
and another with clearing the status bit indicating a device is configured
(when resetting the device the configuration remains across the reset so
the bit should not be cleared, otherwise, the necessary services will not
be re-started in adf_dev_start() after the reset - clear the bit only when
actually deleting the configuration).
Signed-off-by: Bruce Allan <bruce.w.allan@intel.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2015-01-09 11:54:58 -08:00
|
|
|
if (hw_data->init_admin_comms && hw_data->init_admin_comms(accel_dev)) {
|
|
|
|
|
dev_err(&GET_DEV(accel_dev), "Failed initialize admin comms\n");
|
|
|
|
|
return -EFAULT;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (hw_data->init_arb && hw_data->init_arb(accel_dev)) {
|
|
|
|
|
dev_err(&GET_DEV(accel_dev), "Failed initialize hw arbiter\n");
|
|
|
|
|
return -EFAULT;
|
|
|
|
|
}
|
|
|
|
|
|
2023-10-20 15:49:23 +02:00
|
|
|
if (hw_data->get_ring_to_svc_map)
|
|
|
|
|
hw_data->ring_to_svc_map = hw_data->get_ring_to_svc_map(accel_dev);
|
|
|
|
|
|
2014-06-05 13:42:39 -07:00
|
|
|
if (adf_ae_init(accel_dev)) {
|
2015-03-19 16:03:44 -07:00
|
|
|
dev_err(&GET_DEV(accel_dev),
|
|
|
|
|
"Failed to initialise Acceleration Engine\n");
|
2014-06-05 13:42:39 -07:00
|
|
|
return -EFAULT;
|
|
|
|
|
}
|
|
|
|
|
set_bit(ADF_STATUS_AE_INITIALISED, &accel_dev->status);
|
|
|
|
|
|
|
|
|
|
if (adf_ae_fw_load(accel_dev)) {
|
2015-03-19 16:03:44 -07:00
|
|
|
dev_err(&GET_DEV(accel_dev),
|
|
|
|
|
"Failed to load acceleration FW\n");
|
2014-06-05 13:42:39 -07:00
|
|
|
return -EFAULT;
|
|
|
|
|
}
|
|
|
|
|
set_bit(ADF_STATUS_AE_UCODE_LOADED, &accel_dev->status);
|
|
|
|
|
|
|
|
|
|
if (hw_data->alloc_irq(accel_dev)) {
|
2015-03-19 16:03:44 -07:00
|
|
|
dev_err(&GET_DEV(accel_dev), "Failed to allocate interrupts\n");
|
2014-06-05 13:42:39 -07:00
|
|
|
return -EFAULT;
|
|
|
|
|
}
|
|
|
|
|
set_bit(ADF_STATUS_IRQ_ALLOCATED, &accel_dev->status);
|
|
|
|
|
|
2023-10-20 11:32:45 +01:00
|
|
|
if (hw_data->ras_ops.enable_ras_errors)
|
|
|
|
|
hw_data->ras_ops.enable_ras_errors(accel_dev);
|
|
|
|
|
|
2021-08-12 21:21:12 +01:00
|
|
|
hw_data->enable_ints(accel_dev);
|
2021-08-12 21:21:24 +01:00
|
|
|
hw_data->enable_error_correction(accel_dev);
|
|
|
|
|
|
2021-11-17 14:30:47 +00:00
|
|
|
ret = hw_data->pfvf_ops.enable_comms(accel_dev);
|
2021-08-12 21:21:24 +01:00
|
|
|
if (ret)
|
|
|
|
|
return ret;
|
2021-08-12 21:21:12 +01:00
|
|
|
|
2021-12-16 09:13:30 +00:00
|
|
|
if (!test_bit(ADF_STATUS_CONFIGURED, &accel_dev->status) &&
|
|
|
|
|
accel_dev->is_vf) {
|
|
|
|
|
if (qat_crypto_vf_dev_config(accel_dev))
|
|
|
|
|
return -EFAULT;
|
|
|
|
|
}
|
|
|
|
|
|
2023-06-30 19:03:57 +02:00
|
|
|
adf_heartbeat_init(accel_dev);
|
2023-10-20 15:49:28 +02:00
|
|
|
ret = adf_rl_init(accel_dev);
|
|
|
|
|
if (ret && ret != -EOPNOTSUPP)
|
|
|
|
|
return ret;
|
2023-06-30 19:03:57 +02:00
|
|
|
|
crypto: qat - add support for device telemetry
Expose through debugfs device telemetry data for QAT GEN4 devices.
This allows to gather metrics about the performance and the utilization
of a device. In particular, statistics on (1) the utilization of the
PCIe channel, (2) address translation, when SVA is enabled and (3) the
internal engines for crypto and data compression.
If telemetry is supported by the firmware, the driver allocates a DMA
region and a circular buffer. When telemetry is enabled, through the
`control` attribute in debugfs, the driver sends to the firmware, via
the admin interface, the `TL_START` command. This triggers the device to
periodically gather telemetry data from hardware registers and write it
into the DMA memory region. The device writes into the shared region
every second.
The driver, every 500ms, snapshots the DMA shared region into the
circular buffer. This is then used to compute basic metric
(min/max/average) on each counter, every time the `device_data` attribute
is queried.
Telemetry counters are exposed through debugfs in the folder
/sys/kernel/debug/qat_<device>_<BDF>/telemetry.
For details, refer to debugfs-driver-qat_telemetry in Documentation/ABI.
This patch is based on earlier work done by Wojciech Ziemba.
Signed-off-by: Lucas Segarra Fernandez <lucas.segarra.fernandez@intel.com>
Reviewed-by: Giovanni Cabiddu <giovanni.cabiddu@intel.com>
Reviewed-by: Damian Muszynski <damian.muszynski@intel.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2023-12-22 11:35:07 +01:00
|
|
|
ret = adf_tl_init(accel_dev);
|
|
|
|
|
if (ret && ret != -EOPNOTSUPP)
|
|
|
|
|
return ret;
|
|
|
|
|
|
2014-06-05 13:42:39 -07:00
|
|
|
/*
|
|
|
|
|
* Subservice initialisation is divided into two stages: init and start.
|
|
|
|
|
* This is to facilitate any ordering dependencies between services
|
|
|
|
|
* prior to starting any of the accelerators.
|
|
|
|
|
*/
|
2023-08-30 15:54:51 +08:00
|
|
|
list_for_each_entry(service, &service_table, list) {
|
2014-06-05 13:42:39 -07:00
|
|
|
if (service->event_hld(accel_dev, ADF_EVENT_INIT)) {
|
2015-03-19 16:03:44 -07:00
|
|
|
dev_err(&GET_DEV(accel_dev),
|
|
|
|
|
"Failed to initialise service %s\n",
|
|
|
|
|
service->name);
|
2014-06-05 13:42:39 -07:00
|
|
|
return -EFAULT;
|
|
|
|
|
}
|
2016-12-22 15:01:02 +00:00
|
|
|
set_bit(accel_dev->accel_id, service->init_status);
|
2014-06-05 13:42:39 -07:00
|
|
|
}
|
|
|
|
|
|
2021-08-12 21:21:24 +01:00
|
|
|
return 0;
|
crypto: qat - fix device reset flow
When the device needs a reset, e.g. when an uncorrectable PCIe AER event
occurs, various services/data structures need to be cleaned up, the
hardware reset and the services/data structures initialized and started.
The code to perform the cleanup and initialization was not performed when
a device reset was done.
This patch moves some of the initialization code out of the .probe entry-
point into a separate function that is now called during probe as well as
after the hardware has been reset. Similarly, a new function is added for
first cleaning up these services/data structures prior to resetting. The
new functions are adf_dev_init() and adf_dev_shutdown(), respectively, for
which there are already prototypes but no actual functions just yet and are
now called when the device is reset and during probe/cleanup of the driver.
The down and up flows via ioctl calls has similarly been updated.
In addition, there are two other bugs in the reset flow - one in the logic
for determining whether to schedule a device reset upon receiving an
uncorrectable AER event which prevents the reset flow from being initiated,
and another with clearing the status bit indicating a device is configured
(when resetting the device the configuration remains across the reset so
the bit should not be cleared, otherwise, the necessary services will not
be re-started in adf_dev_start() after the reset - clear the bit only when
actually deleting the configuration).
Signed-off-by: Bruce Allan <bruce.w.allan@intel.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2015-01-09 11:54:58 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* adf_dev_start() - Start acceleration service for the given accel device
|
|
|
|
|
* @accel_dev: Pointer to acceleration device.
|
|
|
|
|
*
|
|
|
|
|
* Function notifies all the registered services that the acceleration device
|
|
|
|
|
* is ready to be used.
|
|
|
|
|
* To be used by QAT device specific drivers.
|
|
|
|
|
*
|
2015-07-24 13:18:26 -07:00
|
|
|
* Return: 0 on success, error code otherwise.
|
crypto: qat - fix device reset flow
When the device needs a reset, e.g. when an uncorrectable PCIe AER event
occurs, various services/data structures need to be cleaned up, the
hardware reset and the services/data structures initialized and started.
The code to perform the cleanup and initialization was not performed when
a device reset was done.
This patch moves some of the initialization code out of the .probe entry-
point into a separate function that is now called during probe as well as
after the hardware has been reset. Similarly, a new function is added for
first cleaning up these services/data structures prior to resetting. The
new functions are adf_dev_init() and adf_dev_shutdown(), respectively, for
which there are already prototypes but no actual functions just yet and are
now called when the device is reset and during probe/cleanup of the driver.
The down and up flows via ioctl calls has similarly been updated.
In addition, there are two other bugs in the reset flow - one in the logic
for determining whether to schedule a device reset upon receiving an
uncorrectable AER event which prevents the reset flow from being initiated,
and another with clearing the status bit indicating a device is configured
(when resetting the device the configuration remains across the reset so
the bit should not be cleared, otherwise, the necessary services will not
be re-started in adf_dev_start() after the reset - clear the bit only when
actually deleting the configuration).
Signed-off-by: Bruce Allan <bruce.w.allan@intel.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2015-01-09 11:54:58 -08:00
|
|
|
*/
|
2023-02-27 15:55:45 -05:00
|
|
|
static int adf_dev_start(struct adf_accel_dev *accel_dev)
|
crypto: qat - fix device reset flow
When the device needs a reset, e.g. when an uncorrectable PCIe AER event
occurs, various services/data structures need to be cleaned up, the
hardware reset and the services/data structures initialized and started.
The code to perform the cleanup and initialization was not performed when
a device reset was done.
This patch moves some of the initialization code out of the .probe entry-
point into a separate function that is now called during probe as well as
after the hardware has been reset. Similarly, a new function is added for
first cleaning up these services/data structures prior to resetting. The
new functions are adf_dev_init() and adf_dev_shutdown(), respectively, for
which there are already prototypes but no actual functions just yet and are
now called when the device is reset and during probe/cleanup of the driver.
The down and up flows via ioctl calls has similarly been updated.
In addition, there are two other bugs in the reset flow - one in the logic
for determining whether to schedule a device reset upon receiving an
uncorrectable AER event which prevents the reset flow from being initiated,
and another with clearing the status bit indicating a device is configured
(when resetting the device the configuration remains across the reset so
the bit should not be cleared, otherwise, the necessary services will not
be re-started in adf_dev_start() after the reset - clear the bit only when
actually deleting the configuration).
Signed-off-by: Bruce Allan <bruce.w.allan@intel.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2015-01-09 11:54:58 -08:00
|
|
|
{
|
2015-08-07 11:34:20 -07:00
|
|
|
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
|
crypto: qat - fix device reset flow
When the device needs a reset, e.g. when an uncorrectable PCIe AER event
occurs, various services/data structures need to be cleaned up, the
hardware reset and the services/data structures initialized and started.
The code to perform the cleanup and initialization was not performed when
a device reset was done.
This patch moves some of the initialization code out of the .probe entry-
point into a separate function that is now called during probe as well as
after the hardware has been reset. Similarly, a new function is added for
first cleaning up these services/data structures prior to resetting. The
new functions are adf_dev_init() and adf_dev_shutdown(), respectively, for
which there are already prototypes but no actual functions just yet and are
now called when the device is reset and during probe/cleanup of the driver.
The down and up flows via ioctl calls has similarly been updated.
In addition, there are two other bugs in the reset flow - one in the logic
for determining whether to schedule a device reset upon receiving an
uncorrectable AER event which prevents the reset flow from being initiated,
and another with clearing the status bit indicating a device is configured
(when resetting the device the configuration remains across the reset so
the bit should not be cleared, otherwise, the necessary services will not
be re-started in adf_dev_start() after the reset - clear the bit only when
actually deleting the configuration).
Signed-off-by: Bruce Allan <bruce.w.allan@intel.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2015-01-09 11:54:58 -08:00
|
|
|
struct service_hndl *service;
|
2023-06-30 19:03:54 +02:00
|
|
|
int ret;
|
crypto: qat - fix device reset flow
When the device needs a reset, e.g. when an uncorrectable PCIe AER event
occurs, various services/data structures need to be cleaned up, the
hardware reset and the services/data structures initialized and started.
The code to perform the cleanup and initialization was not performed when
a device reset was done.
This patch moves some of the initialization code out of the .probe entry-
point into a separate function that is now called during probe as well as
after the hardware has been reset. Similarly, a new function is added for
first cleaning up these services/data structures prior to resetting. The
new functions are adf_dev_init() and adf_dev_shutdown(), respectively, for
which there are already prototypes but no actual functions just yet and are
now called when the device is reset and during probe/cleanup of the driver.
The down and up flows via ioctl calls has similarly been updated.
In addition, there are two other bugs in the reset flow - one in the logic
for determining whether to schedule a device reset upon receiving an
uncorrectable AER event which prevents the reset flow from being initiated,
and another with clearing the status bit indicating a device is configured
(when resetting the device the configuration remains across the reset so
the bit should not be cleared, otherwise, the necessary services will not
be re-started in adf_dev_start() after the reset - clear the bit only when
actually deleting the configuration).
Signed-off-by: Bruce Allan <bruce.w.allan@intel.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2015-01-09 11:54:58 -08:00
|
|
|
|
|
|
|
|
set_bit(ADF_STATUS_STARTING, &accel_dev->status);
|
|
|
|
|
|
2014-06-05 13:42:39 -07:00
|
|
|
if (adf_ae_start(accel_dev)) {
|
2015-03-19 16:03:44 -07:00
|
|
|
dev_err(&GET_DEV(accel_dev), "AE Start Failed\n");
|
2014-06-05 13:42:39 -07:00
|
|
|
return -EFAULT;
|
|
|
|
|
}
|
|
|
|
|
set_bit(ADF_STATUS_AE_STARTED, &accel_dev->status);
|
|
|
|
|
|
2015-08-07 11:34:20 -07:00
|
|
|
if (hw_data->send_admin_init(accel_dev)) {
|
|
|
|
|
dev_err(&GET_DEV(accel_dev), "Failed to send init message\n");
|
|
|
|
|
return -EFAULT;
|
2014-06-05 13:42:39 -07:00
|
|
|
}
|
2015-08-07 11:34:20 -07:00
|
|
|
|
2023-06-30 19:03:56 +02:00
|
|
|
if (hw_data->measure_clock) {
|
|
|
|
|
ret = hw_data->measure_clock(accel_dev);
|
|
|
|
|
if (ret) {
|
|
|
|
|
dev_err(&GET_DEV(accel_dev), "Failed measure device clock\n");
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2021-04-09 14:56:19 +01:00
|
|
|
/* Set ssm watch dog timer */
|
|
|
|
|
if (hw_data->set_ssm_wdtimer)
|
|
|
|
|
hw_data->set_ssm_wdtimer(accel_dev);
|
|
|
|
|
|
2022-02-10 13:38:27 +00:00
|
|
|
/* Enable Power Management */
|
|
|
|
|
if (hw_data->enable_pm && hw_data->enable_pm(accel_dev)) {
|
|
|
|
|
dev_err(&GET_DEV(accel_dev), "Failed to configure Power Management\n");
|
|
|
|
|
return -EFAULT;
|
|
|
|
|
}
|
|
|
|
|
|
2023-06-30 19:03:54 +02:00
|
|
|
if (hw_data->start_timer) {
|
|
|
|
|
ret = hw_data->start_timer(accel_dev);
|
|
|
|
|
if (ret) {
|
|
|
|
|
dev_err(&GET_DEV(accel_dev), "Failed to start internal sync timer\n");
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2023-06-30 19:03:57 +02:00
|
|
|
adf_heartbeat_start(accel_dev);
|
2023-10-20 15:49:28 +02:00
|
|
|
ret = adf_rl_start(accel_dev);
|
|
|
|
|
if (ret && ret != -EOPNOTSUPP)
|
|
|
|
|
return ret;
|
2023-06-30 19:03:57 +02:00
|
|
|
|
crypto: qat - add support for device telemetry
Expose through debugfs device telemetry data for QAT GEN4 devices.
This allows to gather metrics about the performance and the utilization
of a device. In particular, statistics on (1) the utilization of the
PCIe channel, (2) address translation, when SVA is enabled and (3) the
internal engines for crypto and data compression.
If telemetry is supported by the firmware, the driver allocates a DMA
region and a circular buffer. When telemetry is enabled, through the
`control` attribute in debugfs, the driver sends to the firmware, via
the admin interface, the `TL_START` command. This triggers the device to
periodically gather telemetry data from hardware registers and write it
into the DMA memory region. The device writes into the shared region
every second.
The driver, every 500ms, snapshots the DMA shared region into the
circular buffer. This is then used to compute basic metric
(min/max/average) on each counter, every time the `device_data` attribute
is queried.
Telemetry counters are exposed through debugfs in the folder
/sys/kernel/debug/qat_<device>_<BDF>/telemetry.
For details, refer to debugfs-driver-qat_telemetry in Documentation/ABI.
This patch is based on earlier work done by Wojciech Ziemba.
Signed-off-by: Lucas Segarra Fernandez <lucas.segarra.fernandez@intel.com>
Reviewed-by: Giovanni Cabiddu <giovanni.cabiddu@intel.com>
Reviewed-by: Damian Muszynski <damian.muszynski@intel.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2023-12-22 11:35:07 +01:00
|
|
|
ret = adf_tl_start(accel_dev);
|
|
|
|
|
if (ret && ret != -EOPNOTSUPP)
|
|
|
|
|
return ret;
|
|
|
|
|
|
2023-08-30 15:54:51 +08:00
|
|
|
list_for_each_entry(service, &service_table, list) {
|
2014-06-05 13:42:39 -07:00
|
|
|
if (service->event_hld(accel_dev, ADF_EVENT_START)) {
|
2015-03-19 16:03:44 -07:00
|
|
|
dev_err(&GET_DEV(accel_dev),
|
|
|
|
|
"Failed to start service %s\n",
|
|
|
|
|
service->name);
|
2014-06-05 13:42:39 -07:00
|
|
|
return -EFAULT;
|
|
|
|
|
}
|
2016-12-22 15:01:02 +00:00
|
|
|
set_bit(accel_dev->accel_id, service->start_status);
|
2014-06-05 13:42:39 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
clear_bit(ADF_STATUS_STARTING, &accel_dev->status);
|
|
|
|
|
set_bit(ADF_STATUS_STARTED, &accel_dev->status);
|
|
|
|
|
|
2015-08-07 11:34:25 -07:00
|
|
|
if (!list_empty(&accel_dev->crypto_list) &&
|
|
|
|
|
(qat_algs_register() || qat_asym_algs_register())) {
|
2015-03-19 16:03:44 -07:00
|
|
|
dev_err(&GET_DEV(accel_dev),
|
|
|
|
|
"Failed to register crypto algs\n");
|
2014-06-05 13:42:39 -07:00
|
|
|
set_bit(ADF_STATUS_STARTING, &accel_dev->status);
|
|
|
|
|
clear_bit(ADF_STATUS_STARTED, &accel_dev->status);
|
|
|
|
|
return -EFAULT;
|
|
|
|
|
}
|
crypto: qat - fix unregistration of crypto algorithms
The function adf_dev_init(), through the subsystem qat_crypto, populates
the list of list of crypto instances accel_dev->crypto_list.
If the list of instances is not empty, the function adf_dev_start() will
then call qat_algs_registers() and qat_asym_algs_register() to register
the crypto algorithms into the crypto framework.
If any of the functions in adf_dev_start() fail, the caller of such
function, in the error path calls adf_dev_down() which in turn call
adf_dev_stop() and adf_dev_shutdown(), see for example the function
state_store in adf_sriov.c.
However, if the registration of crypto algorithms is not done,
adf_dev_stop() will try to unregister the algorithms regardless.
This might cause the counter active_devs in qat_algs.c and
qat_asym_algs.c to get to a negative value.
Add a new state, ADF_STATUS_CRYPTO_ALGS_REGISTERED, which tracks if the
crypto algorithms are registered into the crypto framework. Then use
this to unregister the algorithms if such flag is set. This ensures that
the crypto algorithms are only unregistered if previously registered.
Fixes: d8cba25d2c68 ("crypto: qat - Intel(R) QAT driver framework")
Signed-off-by: Giovanni Cabiddu <giovanni.cabiddu@intel.com>
Reviewed-by: Adam Guerin <adam.guerin@intel.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2023-09-14 10:55:48 +01:00
|
|
|
set_bit(ADF_STATUS_CRYPTO_ALGS_REGISTERED, &accel_dev->status);
|
2022-11-28 12:21:20 +00:00
|
|
|
|
|
|
|
|
if (!list_empty(&accel_dev->compression_list) && qat_comp_algs_register()) {
|
|
|
|
|
dev_err(&GET_DEV(accel_dev),
|
|
|
|
|
"Failed to register compression algs\n");
|
|
|
|
|
set_bit(ADF_STATUS_STARTING, &accel_dev->status);
|
|
|
|
|
clear_bit(ADF_STATUS_STARTED, &accel_dev->status);
|
|
|
|
|
return -EFAULT;
|
|
|
|
|
}
|
crypto: qat - fix unregistration of compression algorithms
The function adf_dev_init(), through the subsystem qat_compression,
populates the list of list of compression instances
accel_dev->compression_list. If the list of instances is not empty,
the function adf_dev_start() will then call qat_compression_registers()
register the compression algorithms into the crypto framework.
If any of the functions in adf_dev_start() fail, the caller of such
function, in the error path calls adf_dev_down() which in turn call
adf_dev_stop() and adf_dev_shutdown(), see for example the function
state_store in adf_sriov.c.
However, if the registration of compression algorithms is not done,
adf_dev_stop() will try to unregister the algorithms regardless.
This might cause the counter active_devs in qat_compression.c to get
to a negative value.
Add a new state, ADF_STATUS_COMPRESSION_ALGS_REGISTERED, which tracks
if the compression algorithms are registered into the crypto framework.
Then use this to unregister the algorithms if such flag is set. This
ensures that the compression algorithms are only unregistered if
previously registered.
Fixes: 1198ae56c9a5 ("crypto: qat - expose deflate through acomp api for QAT GEN2")
Signed-off-by: Giovanni Cabiddu <giovanni.cabiddu@intel.com>
Reviewed-by: Adam Guerin <adam.guerin@intel.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2023-09-14 10:55:49 +01:00
|
|
|
set_bit(ADF_STATUS_COMP_ALGS_REGISTERED, &accel_dev->status);
|
2023-05-26 17:48:59 +01:00
|
|
|
|
|
|
|
|
adf_dbgfs_add(accel_dev);
|
2023-10-20 11:32:52 +01:00
|
|
|
adf_sysfs_start_ras(accel_dev);
|
2023-05-26 17:48:59 +01:00
|
|
|
|
2014-06-05 13:42:39 -07:00
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* adf_dev_stop() - Stop acceleration service for the given accel device
|
|
|
|
|
* @accel_dev: Pointer to acceleration device.
|
|
|
|
|
*
|
|
|
|
|
* Function notifies all the registered services that the acceleration device
|
|
|
|
|
* is shuting down.
|
|
|
|
|
* To be used by QAT device specific drivers.
|
|
|
|
|
*
|
2016-03-29 10:21:07 -07:00
|
|
|
* Return: void
|
2014-06-05 13:42:39 -07:00
|
|
|
*/
|
2023-02-27 15:55:45 -05:00
|
|
|
static void adf_dev_stop(struct adf_accel_dev *accel_dev)
|
2014-06-05 13:42:39 -07:00
|
|
|
{
|
2023-06-30 19:03:54 +02:00
|
|
|
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
|
2014-06-05 13:42:39 -07:00
|
|
|
struct service_hndl *service;
|
2015-01-09 11:55:04 -08:00
|
|
|
bool wait = false;
|
|
|
|
|
int ret;
|
2014-06-05 13:42:39 -07:00
|
|
|
|
|
|
|
|
if (!adf_dev_started(accel_dev) &&
|
2016-03-29 10:21:07 -07:00
|
|
|
!test_bit(ADF_STATUS_STARTING, &accel_dev->status))
|
|
|
|
|
return;
|
|
|
|
|
|
crypto: qat - add support for device telemetry
Expose through debugfs device telemetry data for QAT GEN4 devices.
This allows to gather metrics about the performance and the utilization
of a device. In particular, statistics on (1) the utilization of the
PCIe channel, (2) address translation, when SVA is enabled and (3) the
internal engines for crypto and data compression.
If telemetry is supported by the firmware, the driver allocates a DMA
region and a circular buffer. When telemetry is enabled, through the
`control` attribute in debugfs, the driver sends to the firmware, via
the admin interface, the `TL_START` command. This triggers the device to
periodically gather telemetry data from hardware registers and write it
into the DMA memory region. The device writes into the shared region
every second.
The driver, every 500ms, snapshots the DMA shared region into the
circular buffer. This is then used to compute basic metric
(min/max/average) on each counter, every time the `device_data` attribute
is queried.
Telemetry counters are exposed through debugfs in the folder
/sys/kernel/debug/qat_<device>_<BDF>/telemetry.
For details, refer to debugfs-driver-qat_telemetry in Documentation/ABI.
This patch is based on earlier work done by Wojciech Ziemba.
Signed-off-by: Lucas Segarra Fernandez <lucas.segarra.fernandez@intel.com>
Reviewed-by: Giovanni Cabiddu <giovanni.cabiddu@intel.com>
Reviewed-by: Damian Muszynski <damian.muszynski@intel.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2023-12-22 11:35:07 +01:00
|
|
|
adf_tl_stop(accel_dev);
|
2023-10-20 15:49:28 +02:00
|
|
|
adf_rl_stop(accel_dev);
|
2023-05-26 17:48:59 +01:00
|
|
|
adf_dbgfs_rm(accel_dev);
|
2023-10-20 11:32:52 +01:00
|
|
|
adf_sysfs_stop_ras(accel_dev);
|
2023-05-26 17:48:59 +01:00
|
|
|
|
2014-06-05 13:42:39 -07:00
|
|
|
clear_bit(ADF_STATUS_STARTING, &accel_dev->status);
|
|
|
|
|
clear_bit(ADF_STATUS_STARTED, &accel_dev->status);
|
|
|
|
|
|
crypto: qat - fix unregistration of crypto algorithms
The function adf_dev_init(), through the subsystem qat_crypto, populates
the list of list of crypto instances accel_dev->crypto_list.
If the list of instances is not empty, the function adf_dev_start() will
then call qat_algs_registers() and qat_asym_algs_register() to register
the crypto algorithms into the crypto framework.
If any of the functions in adf_dev_start() fail, the caller of such
function, in the error path calls adf_dev_down() which in turn call
adf_dev_stop() and adf_dev_shutdown(), see for example the function
state_store in adf_sriov.c.
However, if the registration of crypto algorithms is not done,
adf_dev_stop() will try to unregister the algorithms regardless.
This might cause the counter active_devs in qat_algs.c and
qat_asym_algs.c to get to a negative value.
Add a new state, ADF_STATUS_CRYPTO_ALGS_REGISTERED, which tracks if the
crypto algorithms are registered into the crypto framework. Then use
this to unregister the algorithms if such flag is set. This ensures that
the crypto algorithms are only unregistered if previously registered.
Fixes: d8cba25d2c68 ("crypto: qat - Intel(R) QAT driver framework")
Signed-off-by: Giovanni Cabiddu <giovanni.cabiddu@intel.com>
Reviewed-by: Adam Guerin <adam.guerin@intel.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2023-09-14 10:55:48 +01:00
|
|
|
if (!list_empty(&accel_dev->crypto_list) &&
|
|
|
|
|
test_bit(ADF_STATUS_CRYPTO_ALGS_REGISTERED, &accel_dev->status)) {
|
2015-09-22 11:57:47 -07:00
|
|
|
qat_algs_unregister();
|
2015-08-07 11:34:25 -07:00
|
|
|
qat_asym_algs_unregister();
|
2015-09-22 11:57:47 -07:00
|
|
|
}
|
crypto: qat - fix unregistration of crypto algorithms
The function adf_dev_init(), through the subsystem qat_crypto, populates
the list of list of crypto instances accel_dev->crypto_list.
If the list of instances is not empty, the function adf_dev_start() will
then call qat_algs_registers() and qat_asym_algs_register() to register
the crypto algorithms into the crypto framework.
If any of the functions in adf_dev_start() fail, the caller of such
function, in the error path calls adf_dev_down() which in turn call
adf_dev_stop() and adf_dev_shutdown(), see for example the function
state_store in adf_sriov.c.
However, if the registration of crypto algorithms is not done,
adf_dev_stop() will try to unregister the algorithms regardless.
This might cause the counter active_devs in qat_algs.c and
qat_asym_algs.c to get to a negative value.
Add a new state, ADF_STATUS_CRYPTO_ALGS_REGISTERED, which tracks if the
crypto algorithms are registered into the crypto framework. Then use
this to unregister the algorithms if such flag is set. This ensures that
the crypto algorithms are only unregistered if previously registered.
Fixes: d8cba25d2c68 ("crypto: qat - Intel(R) QAT driver framework")
Signed-off-by: Giovanni Cabiddu <giovanni.cabiddu@intel.com>
Reviewed-by: Adam Guerin <adam.guerin@intel.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2023-09-14 10:55:48 +01:00
|
|
|
clear_bit(ADF_STATUS_CRYPTO_ALGS_REGISTERED, &accel_dev->status);
|
2015-07-15 15:28:38 -07:00
|
|
|
|
crypto: qat - fix unregistration of compression algorithms
The function adf_dev_init(), through the subsystem qat_compression,
populates the list of list of compression instances
accel_dev->compression_list. If the list of instances is not empty,
the function adf_dev_start() will then call qat_compression_registers()
register the compression algorithms into the crypto framework.
If any of the functions in adf_dev_start() fail, the caller of such
function, in the error path calls adf_dev_down() which in turn call
adf_dev_stop() and adf_dev_shutdown(), see for example the function
state_store in adf_sriov.c.
However, if the registration of compression algorithms is not done,
adf_dev_stop() will try to unregister the algorithms regardless.
This might cause the counter active_devs in qat_compression.c to get
to a negative value.
Add a new state, ADF_STATUS_COMPRESSION_ALGS_REGISTERED, which tracks
if the compression algorithms are registered into the crypto framework.
Then use this to unregister the algorithms if such flag is set. This
ensures that the compression algorithms are only unregistered if
previously registered.
Fixes: 1198ae56c9a5 ("crypto: qat - expose deflate through acomp api for QAT GEN2")
Signed-off-by: Giovanni Cabiddu <giovanni.cabiddu@intel.com>
Reviewed-by: Adam Guerin <adam.guerin@intel.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2023-09-14 10:55:49 +01:00
|
|
|
if (!list_empty(&accel_dev->compression_list) &&
|
|
|
|
|
test_bit(ADF_STATUS_COMP_ALGS_REGISTERED, &accel_dev->status))
|
2022-11-28 12:21:20 +00:00
|
|
|
qat_comp_algs_unregister();
|
crypto: qat - fix unregistration of compression algorithms
The function adf_dev_init(), through the subsystem qat_compression,
populates the list of list of compression instances
accel_dev->compression_list. If the list of instances is not empty,
the function adf_dev_start() will then call qat_compression_registers()
register the compression algorithms into the crypto framework.
If any of the functions in adf_dev_start() fail, the caller of such
function, in the error path calls adf_dev_down() which in turn call
adf_dev_stop() and adf_dev_shutdown(), see for example the function
state_store in adf_sriov.c.
However, if the registration of compression algorithms is not done,
adf_dev_stop() will try to unregister the algorithms regardless.
This might cause the counter active_devs in qat_compression.c to get
to a negative value.
Add a new state, ADF_STATUS_COMPRESSION_ALGS_REGISTERED, which tracks
if the compression algorithms are registered into the crypto framework.
Then use this to unregister the algorithms if such flag is set. This
ensures that the compression algorithms are only unregistered if
previously registered.
Fixes: 1198ae56c9a5 ("crypto: qat - expose deflate through acomp api for QAT GEN2")
Signed-off-by: Giovanni Cabiddu <giovanni.cabiddu@intel.com>
Reviewed-by: Adam Guerin <adam.guerin@intel.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2023-09-14 10:55:49 +01:00
|
|
|
clear_bit(ADF_STATUS_COMP_ALGS_REGISTERED, &accel_dev->status);
|
2022-11-28 12:21:20 +00:00
|
|
|
|
2023-08-30 15:54:51 +08:00
|
|
|
list_for_each_entry(service, &service_table, list) {
|
2016-12-22 15:01:02 +00:00
|
|
|
if (!test_bit(accel_dev->accel_id, service->start_status))
|
2014-06-05 13:42:39 -07:00
|
|
|
continue;
|
|
|
|
|
ret = service->event_hld(accel_dev, ADF_EVENT_STOP);
|
|
|
|
|
if (!ret) {
|
2016-12-22 15:01:02 +00:00
|
|
|
clear_bit(accel_dev->accel_id, service->start_status);
|
2014-06-05 13:42:39 -07:00
|
|
|
} else if (ret == -EAGAIN) {
|
2015-01-09 11:55:04 -08:00
|
|
|
wait = true;
|
2016-12-22 15:01:02 +00:00
|
|
|
clear_bit(accel_dev->accel_id, service->start_status);
|
2014-06-05 13:42:39 -07:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2023-06-30 19:03:54 +02:00
|
|
|
if (hw_data->stop_timer)
|
|
|
|
|
hw_data->stop_timer(accel_dev);
|
|
|
|
|
|
2024-07-17 07:44:57 -04:00
|
|
|
hw_data->disable_iov(accel_dev);
|
|
|
|
|
|
2014-06-05 13:42:39 -07:00
|
|
|
if (wait)
|
|
|
|
|
msleep(100);
|
|
|
|
|
|
2015-01-09 11:55:09 -08:00
|
|
|
if (test_bit(ADF_STATUS_AE_STARTED, &accel_dev->status)) {
|
2014-06-05 13:42:39 -07:00
|
|
|
if (adf_ae_stop(accel_dev))
|
2015-03-19 16:03:44 -07:00
|
|
|
dev_err(&GET_DEV(accel_dev), "failed to stop AE\n");
|
2014-06-05 13:42:39 -07:00
|
|
|
else
|
|
|
|
|
clear_bit(ADF_STATUS_AE_STARTED, &accel_dev->status);
|
|
|
|
|
}
|
crypto: qat - fix device reset flow
When the device needs a reset, e.g. when an uncorrectable PCIe AER event
occurs, various services/data structures need to be cleaned up, the
hardware reset and the services/data structures initialized and started.
The code to perform the cleanup and initialization was not performed when
a device reset was done.
This patch moves some of the initialization code out of the .probe entry-
point into a separate function that is now called during probe as well as
after the hardware has been reset. Similarly, a new function is added for
first cleaning up these services/data structures prior to resetting. The
new functions are adf_dev_init() and adf_dev_shutdown(), respectively, for
which there are already prototypes but no actual functions just yet and are
now called when the device is reset and during probe/cleanup of the driver.
The down and up flows via ioctl calls has similarly been updated.
In addition, there are two other bugs in the reset flow - one in the logic
for determining whether to schedule a device reset upon receiving an
uncorrectable AER event which prevents the reset flow from being initiated,
and another with clearing the status bit indicating a device is configured
(when resetting the device the configuration remains across the reset so
the bit should not be cleared, otherwise, the necessary services will not
be re-started in adf_dev_start() after the reset - clear the bit only when
actually deleting the configuration).
Signed-off-by: Bruce Allan <bruce.w.allan@intel.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2015-01-09 11:54:58 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* adf_dev_shutdown() - shutdown acceleration services and data strucutures
|
|
|
|
|
* @accel_dev: Pointer to acceleration device
|
|
|
|
|
*
|
|
|
|
|
* Cleanup the ring data structures and the admin comms and arbitration
|
|
|
|
|
* services.
|
|
|
|
|
*/
|
2023-02-27 15:55:45 -05:00
|
|
|
static void adf_dev_shutdown(struct adf_accel_dev *accel_dev)
|
crypto: qat - fix device reset flow
When the device needs a reset, e.g. when an uncorrectable PCIe AER event
occurs, various services/data structures need to be cleaned up, the
hardware reset and the services/data structures initialized and started.
The code to perform the cleanup and initialization was not performed when
a device reset was done.
This patch moves some of the initialization code out of the .probe entry-
point into a separate function that is now called during probe as well as
after the hardware has been reset. Similarly, a new function is added for
first cleaning up these services/data structures prior to resetting. The
new functions are adf_dev_init() and adf_dev_shutdown(), respectively, for
which there are already prototypes but no actual functions just yet and are
now called when the device is reset and during probe/cleanup of the driver.
The down and up flows via ioctl calls has similarly been updated.
In addition, there are two other bugs in the reset flow - one in the logic
for determining whether to schedule a device reset upon receiving an
uncorrectable AER event which prevents the reset flow from being initiated,
and another with clearing the status bit indicating a device is configured
(when resetting the device the configuration remains across the reset so
the bit should not be cleared, otherwise, the necessary services will not
be re-started in adf_dev_start() after the reset - clear the bit only when
actually deleting the configuration).
Signed-off-by: Bruce Allan <bruce.w.allan@intel.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2015-01-09 11:54:58 -08:00
|
|
|
{
|
|
|
|
|
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
|
|
|
|
|
struct service_hndl *service;
|
|
|
|
|
|
|
|
|
|
if (!hw_data) {
|
|
|
|
|
dev_err(&GET_DEV(accel_dev),
|
|
|
|
|
"QAT: Failed to shutdown device - hw_data not set\n");
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
2014-06-05 13:42:39 -07:00
|
|
|
if (test_bit(ADF_STATUS_AE_UCODE_LOADED, &accel_dev->status)) {
|
2015-04-03 08:41:17 -07:00
|
|
|
adf_ae_fw_release(accel_dev);
|
|
|
|
|
clear_bit(ADF_STATUS_AE_UCODE_LOADED, &accel_dev->status);
|
2014-06-05 13:42:39 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (test_bit(ADF_STATUS_AE_INITIALISED, &accel_dev->status)) {
|
|
|
|
|
if (adf_ae_shutdown(accel_dev))
|
2015-03-19 16:03:44 -07:00
|
|
|
dev_err(&GET_DEV(accel_dev),
|
|
|
|
|
"Failed to shutdown Accel Engine\n");
|
2014-06-05 13:42:39 -07:00
|
|
|
else
|
|
|
|
|
clear_bit(ADF_STATUS_AE_INITIALISED,
|
|
|
|
|
&accel_dev->status);
|
|
|
|
|
}
|
|
|
|
|
|
2023-08-30 15:54:51 +08:00
|
|
|
list_for_each_entry(service, &service_table, list) {
|
2016-12-22 15:01:02 +00:00
|
|
|
if (!test_bit(accel_dev->accel_id, service->init_status))
|
2014-06-05 13:42:39 -07:00
|
|
|
continue;
|
|
|
|
|
if (service->event_hld(accel_dev, ADF_EVENT_SHUTDOWN))
|
2015-03-19 16:03:44 -07:00
|
|
|
dev_err(&GET_DEV(accel_dev),
|
|
|
|
|
"Failed to shutdown service %s\n",
|
|
|
|
|
service->name);
|
2014-06-05 13:42:39 -07:00
|
|
|
else
|
2016-12-22 15:01:02 +00:00
|
|
|
clear_bit(accel_dev->accel_id, service->init_status);
|
2014-06-05 13:42:39 -07:00
|
|
|
}
|
|
|
|
|
|
2023-10-20 15:49:28 +02:00
|
|
|
adf_rl_exit(accel_dev);
|
|
|
|
|
|
2023-10-20 11:32:45 +01:00
|
|
|
if (hw_data->ras_ops.disable_ras_errors)
|
|
|
|
|
hw_data->ras_ops.disable_ras_errors(accel_dev);
|
|
|
|
|
|
2023-06-30 19:03:57 +02:00
|
|
|
adf_heartbeat_shutdown(accel_dev);
|
crypto: qat - add support for device telemetry
Expose through debugfs device telemetry data for QAT GEN4 devices.
This allows to gather metrics about the performance and the utilization
of a device. In particular, statistics on (1) the utilization of the
PCIe channel, (2) address translation, when SVA is enabled and (3) the
internal engines for crypto and data compression.
If telemetry is supported by the firmware, the driver allocates a DMA
region and a circular buffer. When telemetry is enabled, through the
`control` attribute in debugfs, the driver sends to the firmware, via
the admin interface, the `TL_START` command. This triggers the device to
periodically gather telemetry data from hardware registers and write it
into the DMA memory region. The device writes into the shared region
every second.
The driver, every 500ms, snapshots the DMA shared region into the
circular buffer. This is then used to compute basic metric
(min/max/average) on each counter, every time the `device_data` attribute
is queried.
Telemetry counters are exposed through debugfs in the folder
/sys/kernel/debug/qat_<device>_<BDF>/telemetry.
For details, refer to debugfs-driver-qat_telemetry in Documentation/ABI.
This patch is based on earlier work done by Wojciech Ziemba.
Signed-off-by: Lucas Segarra Fernandez <lucas.segarra.fernandez@intel.com>
Reviewed-by: Giovanni Cabiddu <giovanni.cabiddu@intel.com>
Reviewed-by: Damian Muszynski <damian.muszynski@intel.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2023-12-22 11:35:07 +01:00
|
|
|
|
|
|
|
|
adf_tl_shutdown(accel_dev);
|
2023-06-30 19:03:57 +02:00
|
|
|
|
2014-06-05 13:42:39 -07:00
|
|
|
if (test_bit(ADF_STATUS_IRQ_ALLOCATED, &accel_dev->status)) {
|
|
|
|
|
hw_data->free_irq(accel_dev);
|
|
|
|
|
clear_bit(ADF_STATUS_IRQ_ALLOCATED, &accel_dev->status);
|
|
|
|
|
}
|
|
|
|
|
|
2024-07-17 07:44:56 -04:00
|
|
|
/* If not restarting, delete all cfg sections except for GENERAL */
|
2014-06-05 13:42:39 -07:00
|
|
|
if (!test_bit(ADF_STATUS_RESTARTING, &accel_dev->status))
|
2024-07-17 07:44:56 -04:00
|
|
|
adf_cfg_del_all_except(accel_dev, ADF_GENERAL_SEC);
|
2014-06-05 13:42:39 -07:00
|
|
|
|
crypto: qat - fix device reset flow
When the device needs a reset, e.g. when an uncorrectable PCIe AER event
occurs, various services/data structures need to be cleaned up, the
hardware reset and the services/data structures initialized and started.
The code to perform the cleanup and initialization was not performed when
a device reset was done.
This patch moves some of the initialization code out of the .probe entry-
point into a separate function that is now called during probe as well as
after the hardware has been reset. Similarly, a new function is added for
first cleaning up these services/data structures prior to resetting. The
new functions are adf_dev_init() and adf_dev_shutdown(), respectively, for
which there are already prototypes but no actual functions just yet and are
now called when the device is reset and during probe/cleanup of the driver.
The down and up flows via ioctl calls has similarly been updated.
In addition, there are two other bugs in the reset flow - one in the logic
for determining whether to schedule a device reset upon receiving an
uncorrectable AER event which prevents the reset flow from being initiated,
and another with clearing the status bit indicating a device is configured
(when resetting the device the configuration remains across the reset so
the bit should not be cleared, otherwise, the necessary services will not
be re-started in adf_dev_start() after the reset - clear the bit only when
actually deleting the configuration).
Signed-off-by: Bruce Allan <bruce.w.allan@intel.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2015-01-09 11:54:58 -08:00
|
|
|
if (hw_data->exit_arb)
|
|
|
|
|
hw_data->exit_arb(accel_dev);
|
|
|
|
|
|
|
|
|
|
if (hw_data->exit_admin_comms)
|
|
|
|
|
hw_data->exit_admin_comms(accel_dev);
|
|
|
|
|
|
|
|
|
|
adf_cleanup_etr_data(accel_dev);
|
2025-07-11 13:27:43 +01:00
|
|
|
adf_misc_wq_flush();
|
2015-12-04 16:56:28 -08:00
|
|
|
adf_dev_restore(accel_dev);
|
2014-06-05 13:42:39 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
int adf_dev_restarting_notify(struct adf_accel_dev *accel_dev)
|
|
|
|
|
{
|
|
|
|
|
struct service_hndl *service;
|
|
|
|
|
|
2023-08-30 15:54:51 +08:00
|
|
|
list_for_each_entry(service, &service_table, list) {
|
2014-06-05 13:42:39 -07:00
|
|
|
if (service->event_hld(accel_dev, ADF_EVENT_RESTARTING))
|
2015-03-19 16:03:44 -07:00
|
|
|
dev_err(&GET_DEV(accel_dev),
|
|
|
|
|
"Failed to restart service %s.\n",
|
|
|
|
|
service->name);
|
2014-06-05 13:42:39 -07:00
|
|
|
}
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
int adf_dev_restarted_notify(struct adf_accel_dev *accel_dev)
|
|
|
|
|
{
|
|
|
|
|
struct service_hndl *service;
|
|
|
|
|
|
2023-08-30 15:54:51 +08:00
|
|
|
list_for_each_entry(service, &service_table, list) {
|
2014-06-05 13:42:39 -07:00
|
|
|
if (service->event_hld(accel_dev, ADF_EVENT_RESTARTED))
|
2015-03-19 16:03:44 -07:00
|
|
|
dev_err(&GET_DEV(accel_dev),
|
|
|
|
|
"Failed to restart service %s.\n",
|
|
|
|
|
service->name);
|
2014-06-05 13:42:39 -07:00
|
|
|
}
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
2022-06-27 09:36:51 +01:00
|
|
|
|
2024-02-02 18:53:17 +08:00
|
|
|
void adf_error_notifier(struct adf_accel_dev *accel_dev)
|
|
|
|
|
{
|
|
|
|
|
struct service_hndl *service;
|
|
|
|
|
|
|
|
|
|
list_for_each_entry(service, &service_table, list) {
|
|
|
|
|
if (service->event_hld(accel_dev, ADF_EVENT_FATAL_ERROR))
|
|
|
|
|
dev_err(&GET_DEV(accel_dev),
|
|
|
|
|
"Failed to send error event to %s.\n",
|
|
|
|
|
service->name);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2024-07-17 07:44:56 -04:00
|
|
|
int adf_dev_down(struct adf_accel_dev *accel_dev)
|
2023-02-27 15:55:42 -05:00
|
|
|
{
|
|
|
|
|
int ret = 0;
|
|
|
|
|
|
|
|
|
|
if (!accel_dev)
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
|
|
mutex_lock(&accel_dev->state_lock);
|
|
|
|
|
|
|
|
|
|
adf_dev_stop(accel_dev);
|
|
|
|
|
adf_dev_shutdown(accel_dev);
|
|
|
|
|
|
|
|
|
|
mutex_unlock(&accel_dev->state_lock);
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL_GPL(adf_dev_down);
|
|
|
|
|
|
|
|
|
|
int adf_dev_up(struct adf_accel_dev *accel_dev, bool config)
|
|
|
|
|
{
|
|
|
|
|
int ret = 0;
|
|
|
|
|
|
|
|
|
|
if (!accel_dev)
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
|
|
mutex_lock(&accel_dev->state_lock);
|
|
|
|
|
|
|
|
|
|
if (adf_dev_started(accel_dev)) {
|
|
|
|
|
dev_info(&GET_DEV(accel_dev), "Device qat_dev%d already up\n",
|
|
|
|
|
accel_dev->accel_id);
|
|
|
|
|
ret = -EALREADY;
|
|
|
|
|
goto out;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (config && GET_HW_DATA(accel_dev)->dev_config) {
|
|
|
|
|
ret = GET_HW_DATA(accel_dev)->dev_config(accel_dev);
|
|
|
|
|
if (unlikely(ret))
|
|
|
|
|
goto out;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ret = adf_dev_init(accel_dev);
|
|
|
|
|
if (unlikely(ret))
|
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
|
|
ret = adf_dev_start(accel_dev);
|
|
|
|
|
|
|
|
|
|
out:
|
|
|
|
|
mutex_unlock(&accel_dev->state_lock);
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL_GPL(adf_dev_up);
|
2023-02-27 15:55:44 -05:00
|
|
|
|
|
|
|
|
int adf_dev_restart(struct adf_accel_dev *accel_dev)
|
|
|
|
|
{
|
|
|
|
|
int ret = 0;
|
|
|
|
|
|
|
|
|
|
if (!accel_dev)
|
|
|
|
|
return -EFAULT;
|
|
|
|
|
|
2024-07-17 07:44:56 -04:00
|
|
|
adf_dev_down(accel_dev);
|
2023-02-27 15:55:44 -05:00
|
|
|
|
|
|
|
|
ret = adf_dev_up(accel_dev, false);
|
|
|
|
|
/* if device is already up return success*/
|
|
|
|
|
if (ret == -EALREADY)
|
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL_GPL(adf_dev_restart);
|