Skip to content

Commit 4fefd69

Browse files
awilliamawilliam
committed
Merge branch 'vfio' of https://git.kernel.org/pub/scm/linux/kernel/git/herbert/cryptodev-2.6 into v6.10/vfio/qat-v7
2 parents 848e447 + f0bbfc3 commit 4fefd69

32 files changed

+2655
-387
lines changed

‎drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,14 @@
1010
#include <adf_fw_config.h>
1111
#include <adf_gen4_config.h>
1212
#include <adf_gen4_dc.h>
13+
#include <adf_gen4_hw_csr_data.h>
1314
#include <adf_gen4_hw_data.h>
1415
#include <adf_gen4_pfvf.h>
1516
#include <adf_gen4_pm.h>
1617
#include <adf_gen4_ras.h>
1718
#include <adf_gen4_timer.h>
1819
#include <adf_gen4_tl.h>
20+
#include <adf_gen4_vf_mig.h>
1921
#include "adf_420xx_hw_data.h"
2022
#include "icp_qat_hw.h"
2123

@@ -487,6 +489,7 @@ void adf_init_hw_data_420xx(struct adf_hw_device_data *hw_data, u32 dev_id)
487489
adf_gen4_init_dc_ops(&hw_data->dc_ops);
488490
adf_gen4_init_ras_ops(&hw_data->ras_ops);
489491
adf_gen4_init_tl_data(&hw_data->tl_data);
492+
adf_gen4_init_vf_mig_ops(&hw_data->vfmig_ops);
490493
adf_init_rl_data(&hw_data->rl_data);
491494
}
492495

‎drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,14 @@
1010
#include <adf_fw_config.h>
1111
#include <adf_gen4_config.h>
1212
#include <adf_gen4_dc.h>
13+
#include <adf_gen4_hw_csr_data.h>
1314
#include <adf_gen4_hw_data.h>
1415
#include <adf_gen4_pfvf.h>
1516
#include <adf_gen4_pm.h>
1617
#include "adf_gen4_ras.h"
1718
#include <adf_gen4_timer.h>
1819
#include <adf_gen4_tl.h>
20+
#include <adf_gen4_vf_mig.h>
1921
#include "adf_4xxx_hw_data.h"
2022
#include "icp_qat_hw.h"
2123

@@ -454,6 +456,8 @@ void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data, u32 dev_id)
454456
hw_data->get_ring_to_svc_map = adf_gen4_get_ring_to_svc_map;
455457
hw_data->disable_iov = adf_disable_sriov;
456458
hw_data->ring_pair_reset = adf_gen4_ring_pair_reset;
459+
hw_data->bank_state_save = adf_gen4_bank_state_save;
460+
hw_data->bank_state_restore = adf_gen4_bank_state_restore;
457461
hw_data->enable_pm = adf_gen4_enable_pm;
458462
hw_data->handle_pm_interrupt = adf_gen4_handle_pm_interrupt;
459463
hw_data->dev_config = adf_gen4_dev_config;
@@ -469,6 +473,7 @@ void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data, u32 dev_id)
469473
adf_gen4_init_dc_ops(&hw_data->dc_ops);
470474
adf_gen4_init_ras_ops(&hw_data->ras_ops);
471475
adf_gen4_init_tl_data(&hw_data->tl_data);
476+
adf_gen4_init_vf_mig_ops(&hw_data->vfmig_ops);
472477
adf_init_rl_data(&hw_data->rl_data);
473478
}
474479

‎drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
#include <adf_common_drv.h>
77
#include <adf_gen2_config.h>
88
#include <adf_gen2_dc.h>
9+
#include <adf_gen2_hw_csr_data.h>
910
#include <adf_gen2_hw_data.h>
1011
#include <adf_gen2_pfvf.h>
1112
#include "adf_c3xxx_hw_data.h"

‎drivers/crypto/intel/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
#include <adf_common_drv.h>
55
#include <adf_gen2_config.h>
66
#include <adf_gen2_dc.h>
7+
#include <adf_gen2_hw_csr_data.h>
78
#include <adf_gen2_hw_data.h>
89
#include <adf_gen2_pfvf.h>
910
#include <adf_pfvf_vf_msg.h>

‎drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
#include <adf_common_drv.h>
77
#include <adf_gen2_config.h>
88
#include <adf_gen2_dc.h>
9+
#include <adf_gen2_hw_csr_data.h>
910
#include <adf_gen2_hw_data.h>
1011
#include <adf_gen2_pfvf.h>
1112
#include "adf_c62x_hw_data.h"

‎drivers/crypto/intel/qat/qat_c62xvf/adf_c62xvf_hw_data.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
#include <adf_common_drv.h>
55
#include <adf_gen2_config.h>
66
#include <adf_gen2_dc.h>
7+
#include <adf_gen2_hw_csr_data.h>
78
#include <adf_gen2_hw_data.h>
89
#include <adf_gen2_pfvf.h>
910
#include <adf_pfvf_vf_msg.h>

‎drivers/crypto/intel/qat/qat_common/Makefile

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,16 +14,20 @@ intel_qat-objs := adf_cfg.o \
1414
adf_hw_arbiter.o \
1515
adf_sysfs.o \
1616
adf_sysfs_ras_counters.o \
17+
adf_gen2_hw_csr_data.o \
1718
adf_gen2_hw_data.o \
1819
adf_gen2_config.o \
1920
adf_gen4_config.o \
21+
adf_gen4_hw_csr_data.o \
2022
adf_gen4_hw_data.o \
23+
adf_gen4_vf_mig.o \
2124
adf_gen4_pm.o \
2225
adf_gen2_dc.o \
2326
adf_gen4_dc.o \
2427
adf_gen4_ras.o \
2528
adf_gen4_timer.o \
2629
adf_clock.o \
30+
adf_mstate_mgr.o \
2731
qat_crypto.o \
2832
qat_compression.o \
2933
qat_comp_algs.o \
@@ -52,6 +56,6 @@ intel_qat-$(CONFIG_DEBUG_FS) += adf_transport_debug.o \
5256
intel_qat-$(CONFIG_PCI_IOV) += adf_sriov.o adf_vf_isr.o adf_pfvf_utils.o \
5357
adf_pfvf_pf_msg.o adf_pfvf_pf_proto.o \
5458
adf_pfvf_vf_msg.o adf_pfvf_vf_proto.o \
55-
adf_gen2_pfvf.o adf_gen4_pfvf.o
59+
adf_gen2_pfvf.o adf_gen4_pfvf.o qat_mig_dev.o
5660

5761
intel_qat-$(CONFIG_CRYPTO_DEV_QAT_ERROR_INJECTION) += adf_heartbeat_inject.o

‎drivers/crypto/intel/qat/qat_common/adf_accel_devices.h

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
#include <linux/pci.h>
1010
#include <linux/ratelimit.h>
1111
#include <linux/types.h>
12+
#include <linux/qat/qat_mig_dev.h>
1213
#include "adf_cfg_common.h"
1314
#include "adf_rl.h"
1415
#include "adf_telemetry.h"
@@ -140,6 +141,40 @@ struct admin_info {
140141
u32 mailbox_offset;
141142
};
142143

144+
struct ring_config {
145+
u64 base;
146+
u32 config;
147+
u32 head;
148+
u32 tail;
149+
u32 reserved0;
150+
};
151+
152+
struct bank_state {
153+
u32 ringstat0;
154+
u32 ringstat1;
155+
u32 ringuostat;
156+
u32 ringestat;
157+
u32 ringnestat;
158+
u32 ringnfstat;
159+
u32 ringfstat;
160+
u32 ringcstat0;
161+
u32 ringcstat1;
162+
u32 ringcstat2;
163+
u32 ringcstat3;
164+
u32 iaintflagen;
165+
u32 iaintflagreg;
166+
u32 iaintflagsrcsel0;
167+
u32 iaintflagsrcsel1;
168+
u32 iaintcolen;
169+
u32 iaintcolctl;
170+
u32 iaintflagandcolen;
171+
u32 ringexpstat;
172+
u32 ringexpintenable;
173+
u32 ringsrvarben;
174+
u32 reserved0;
175+
struct ring_config rings[ADF_ETR_MAX_RINGS_PER_BANK];
176+
};
177+
143178
struct adf_hw_csr_ops {
144179
u64 (*build_csr_ring_base_addr)(dma_addr_t addr, u32 size);
145180
u32 (*read_csr_ring_head)(void __iomem *csr_base_addr, u32 bank,
@@ -150,22 +185,49 @@ struct adf_hw_csr_ops {
150185
u32 ring);
151186
void (*write_csr_ring_tail)(void __iomem *csr_base_addr, u32 bank,
152187
u32 ring, u32 value);
188+
u32 (*read_csr_stat)(void __iomem *csr_base_addr, u32 bank);
189+
u32 (*read_csr_uo_stat)(void __iomem *csr_base_addr, u32 bank);
153190
u32 (*read_csr_e_stat)(void __iomem *csr_base_addr, u32 bank);
191+
u32 (*read_csr_ne_stat)(void __iomem *csr_base_addr, u32 bank);
192+
u32 (*read_csr_nf_stat)(void __iomem *csr_base_addr, u32 bank);
193+
u32 (*read_csr_f_stat)(void __iomem *csr_base_addr, u32 bank);
194+
u32 (*read_csr_c_stat)(void __iomem *csr_base_addr, u32 bank);
195+
u32 (*read_csr_exp_stat)(void __iomem *csr_base_addr, u32 bank);
196+
u32 (*read_csr_exp_int_en)(void __iomem *csr_base_addr, u32 bank);
197+
void (*write_csr_exp_int_en)(void __iomem *csr_base_addr, u32 bank,
198+
u32 value);
199+
u32 (*read_csr_ring_config)(void __iomem *csr_base_addr, u32 bank,
200+
u32 ring);
154201
void (*write_csr_ring_config)(void __iomem *csr_base_addr, u32 bank,
155202
u32 ring, u32 value);
203+
dma_addr_t (*read_csr_ring_base)(void __iomem *csr_base_addr, u32 bank,
204+
u32 ring);
156205
void (*write_csr_ring_base)(void __iomem *csr_base_addr, u32 bank,
157206
u32 ring, dma_addr_t addr);
207+
u32 (*read_csr_int_en)(void __iomem *csr_base_addr, u32 bank);
208+
void (*write_csr_int_en)(void __iomem *csr_base_addr, u32 bank,
209+
u32 value);
210+
u32 (*read_csr_int_flag)(void __iomem *csr_base_addr, u32 bank);
158211
void (*write_csr_int_flag)(void __iomem *csr_base_addr, u32 bank,
159212
u32 value);
213+
u32 (*read_csr_int_srcsel)(void __iomem *csr_base_addr, u32 bank);
160214
void (*write_csr_int_srcsel)(void __iomem *csr_base_addr, u32 bank);
215+
void (*write_csr_int_srcsel_w_val)(void __iomem *csr_base_addr,
216+
u32 bank, u32 value);
217+
u32 (*read_csr_int_col_en)(void __iomem *csr_base_addr, u32 bank);
161218
void (*write_csr_int_col_en)(void __iomem *csr_base_addr, u32 bank,
162219
u32 value);
220+
u32 (*read_csr_int_col_ctl)(void __iomem *csr_base_addr, u32 bank);
163221
void (*write_csr_int_col_ctl)(void __iomem *csr_base_addr, u32 bank,
164222
u32 value);
223+
u32 (*read_csr_int_flag_and_col)(void __iomem *csr_base_addr,
224+
u32 bank);
165225
void (*write_csr_int_flag_and_col)(void __iomem *csr_base_addr,
166226
u32 bank, u32 value);
227+
u32 (*read_csr_ring_srv_arb_en)(void __iomem *csr_base_addr, u32 bank);
167228
void (*write_csr_ring_srv_arb_en)(void __iomem *csr_base_addr, u32 bank,
168229
u32 value);
230+
u32 (*get_int_col_ctl_enable_mask)(void);
169231
};
170232

171233
struct adf_cfg_device_data;
@@ -197,6 +259,20 @@ struct adf_dc_ops {
197259
void (*build_deflate_ctx)(void *ctx);
198260
};
199261

262+
struct qat_migdev_ops {
263+
int (*init)(struct qat_mig_dev *mdev);
264+
void (*cleanup)(struct qat_mig_dev *mdev);
265+
void (*reset)(struct qat_mig_dev *mdev);
266+
int (*open)(struct qat_mig_dev *mdev);
267+
void (*close)(struct qat_mig_dev *mdev);
268+
int (*suspend)(struct qat_mig_dev *mdev);
269+
int (*resume)(struct qat_mig_dev *mdev);
270+
int (*save_state)(struct qat_mig_dev *mdev);
271+
int (*save_setup)(struct qat_mig_dev *mdev);
272+
int (*load_state)(struct qat_mig_dev *mdev);
273+
int (*load_setup)(struct qat_mig_dev *mdev, int size);
274+
};
275+
200276
struct adf_dev_err_mask {
201277
u32 cppagentcmdpar_mask;
202278
u32 parerr_ath_cph_mask;
@@ -244,6 +320,10 @@ struct adf_hw_device_data {
244320
void (*enable_ints)(struct adf_accel_dev *accel_dev);
245321
void (*set_ssm_wdtimer)(struct adf_accel_dev *accel_dev);
246322
int (*ring_pair_reset)(struct adf_accel_dev *accel_dev, u32 bank_nr);
323+
int (*bank_state_save)(struct adf_accel_dev *accel_dev, u32 bank_number,
324+
struct bank_state *state);
325+
int (*bank_state_restore)(struct adf_accel_dev *accel_dev,
326+
u32 bank_number, struct bank_state *state);
247327
void (*reset_device)(struct adf_accel_dev *accel_dev);
248328
void (*set_msix_rttable)(struct adf_accel_dev *accel_dev);
249329
const char *(*uof_get_name)(struct adf_accel_dev *accel_dev, u32 obj_num);
@@ -260,6 +340,7 @@ struct adf_hw_device_data {
260340
struct adf_dev_err_mask dev_err_mask;
261341
struct adf_rl_hw_data rl_data;
262342
struct adf_tl_hw_data tl_data;
343+
struct qat_migdev_ops vfmig_ops;
263344
const char *fw_name;
264345
const char *fw_mmp_name;
265346
u32 fuses;
@@ -316,6 +397,7 @@ struct adf_hw_device_data {
316397
#define GET_CSR_OPS(accel_dev) (&(accel_dev)->hw_device->csr_ops)
317398
#define GET_PFVF_OPS(accel_dev) (&(accel_dev)->hw_device->pfvf_ops)
318399
#define GET_DC_OPS(accel_dev) (&(accel_dev)->hw_device->dc_ops)
400+
#define GET_VFMIG_OPS(accel_dev) (&(accel_dev)->hw_device->vfmig_ops)
319401
#define GET_TL_DATA(accel_dev) GET_HW_DATA(accel_dev)->tl_data
320402
#define accel_to_pci_dev(accel_ptr) accel_ptr->accel_pci_dev.pci_dev
321403

@@ -330,11 +412,17 @@ struct adf_fw_loader_data {
330412
struct adf_accel_vf_info {
331413
struct adf_accel_dev *accel_dev;
332414
struct mutex pf2vf_lock; /* protect CSR access for PF2VF messages */
415+
struct mutex pfvf_mig_lock; /* protects PFVF state for migration */
333416
struct ratelimit_state vf2pf_ratelimit;
334417
u32 vf_nr;
335418
bool init;
336419
bool restarting;
337420
u8 vf_compat_ver;
421+
/*
422+
* Private area used for device migration.
423+
* Memory allocation and free is managed by migration driver.
424+
*/
425+
void *mig_priv;
338426
};
339427

340428
struct adf_dc_data {

‎drivers/crypto/intel/qat/qat_common/adf_common_drv.h

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -248,6 +248,16 @@ static inline void __iomem *adf_get_pmisc_base(struct adf_accel_dev *accel_dev)
248248
return pmisc->virt_addr;
249249
}
250250

251+
static inline void __iomem *adf_get_etr_base(struct adf_accel_dev *accel_dev)
252+
{
253+
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
254+
struct adf_bar *etr;
255+
256+
etr = &GET_BARS(accel_dev)[hw_data->get_etr_bar_id(hw_data)];
257+
258+
return etr->virt_addr;
259+
}
260+
251261
static inline void __iomem *adf_get_aram_base(struct adf_accel_dev *accel_dev)
252262
{
253263
struct adf_hw_device_data *hw_data = accel_dev->hw_device;

‎drivers/crypto/intel/qat/qat_common/adf_gen2_hw_csr_data.c

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
// SPDX-License-Identifier: GPL-2.0-only
2+
/* Copyright(c) 2024 Intel Corporation */
3+
#include <linux/types.h>
4+
#include "adf_gen2_hw_csr_data.h"
5+
6+
static u64 build_csr_ring_base_addr(dma_addr_t addr, u32 size)
7+
{
8+
return BUILD_RING_BASE_ADDR(addr, size);
9+
}
10+
11+
static u32 read_csr_ring_head(void __iomem *csr_base_addr, u32 bank, u32 ring)
12+
{
13+
return READ_CSR_RING_HEAD(csr_base_addr, bank, ring);
14+
}
15+
16+
static void write_csr_ring_head(void __iomem *csr_base_addr, u32 bank, u32 ring,
17+
u32 value)
18+
{
19+
WRITE_CSR_RING_HEAD(csr_base_addr, bank, ring, value);
20+
}
21+
22+
static u32 read_csr_ring_tail(void __iomem *csr_base_addr, u32 bank, u32 ring)
23+
{
24+
return READ_CSR_RING_TAIL(csr_base_addr, bank, ring);
25+
}
26+
27+
static void write_csr_ring_tail(void __iomem *csr_base_addr, u32 bank, u32 ring,
28+
u32 value)
29+
{
30+
WRITE_CSR_RING_TAIL(csr_base_addr, bank, ring, value);
31+
}
32+
33+
static u32 read_csr_e_stat(void __iomem *csr_base_addr, u32 bank)
34+
{
35+
return READ_CSR_E_STAT(csr_base_addr, bank);
36+
}
37+
38+
static void write_csr_ring_config(void __iomem *csr_base_addr, u32 bank,
39+
u32 ring, u32 value)
40+
{
41+
WRITE_CSR_RING_CONFIG(csr_base_addr, bank, ring, value);
42+
}
43+
44+
static void write_csr_ring_base(void __iomem *csr_base_addr, u32 bank, u32 ring,
45+
dma_addr_t addr)
46+
{
47+
WRITE_CSR_RING_BASE(csr_base_addr, bank, ring, addr);
48+
}
49+
50+
static void write_csr_int_flag(void __iomem *csr_base_addr, u32 bank, u32 value)
51+
{
52+
WRITE_CSR_INT_FLAG(csr_base_addr, bank, value);
53+
}
54+
55+
static void write_csr_int_srcsel(void __iomem *csr_base_addr, u32 bank)
56+
{
57+
WRITE_CSR_INT_SRCSEL(csr_base_addr, bank);
58+
}
59+
60+
static void write_csr_int_col_en(void __iomem *csr_base_addr, u32 bank,
61+
u32 value)
62+
{
63+
WRITE_CSR_INT_COL_EN(csr_base_addr, bank, value);
64+
}
65+
66+
static void write_csr_int_col_ctl(void __iomem *csr_base_addr, u32 bank,
67+
u32 value)
68+
{
69+
WRITE_CSR_INT_COL_CTL(csr_base_addr, bank, value);
70+
}
71+
72+
static void write_csr_int_flag_and_col(void __iomem *csr_base_addr, u32 bank,
73+
u32 value)
74+
{
75+
WRITE_CSR_INT_FLAG_AND_COL(csr_base_addr, bank, value);
76+
}
77+
78+
static void write_csr_ring_srv_arb_en(void __iomem *csr_base_addr, u32 bank,
79+
u32 value)
80+
{
81+
WRITE_CSR_RING_SRV_ARB_EN(csr_base_addr, bank, value);
82+
}
83+
84+
void adf_gen2_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops)
85+
{
86+
csr_ops->build_csr_ring_base_addr = build_csr_ring_base_addr;
87+
csr_ops->read_csr_ring_head = read_csr_ring_head;
88+
csr_ops->write_csr_ring_head = write_csr_ring_head;
89+
csr_ops->read_csr_ring_tail = read_csr_ring_tail;
90+
csr_ops->write_csr_ring_tail = write_csr_ring_tail;
91+
csr_ops->read_csr_e_stat = read_csr_e_stat;
92+
csr_ops->write_csr_ring_config = write_csr_ring_config;
93+
csr_ops->write_csr_ring_base = write_csr_ring_base;
94+
csr_ops->write_csr_int_flag = write_csr_int_flag;
95+
csr_ops->write_csr_int_srcsel = write_csr_int_srcsel;
96+
csr_ops->write_csr_int_col_en = write_csr_int_col_en;
97+
csr_ops->write_csr_int_col_ctl = write_csr_int_col_ctl;
98+
csr_ops->write_csr_int_flag_and_col = write_csr_int_flag_and_col;
99+
csr_ops->write_csr_ring_srv_arb_en = write_csr_ring_srv_arb_en;
100+
}
101+
EXPORT_SYMBOL_GPL(adf_gen2_init_hw_csr_ops);

0 commit comments

Comments
 (0)