Skip to content

Commit 8a1d824

Browse files
LuBaolujoergroedel
LuBaolu
authored and
joergroedel
committed
iommu/vt-d: Multiple descriptors per qi_submit_sync()
Current qi_submit_sync() only supports single invalidation descriptor per submission and appends wait descriptor after each submission to poll the hardware completion. This extends the qi_submit_sync() helper to support multiple descriptors, and add an option so that the caller could specify the Page-request Drain (PD) bit in the wait descriptor. Signed-off-by: Jacob Pan <[email protected]> Signed-off-by: Lu Baolu <[email protected]> Reviewed-by: Kevin Tian <[email protected]> Link: https://lore.kernel.org/r/[email protected] Signed-off-by: Joerg Roedel <[email protected]>
1 parent 064a57d commit 8a1d824

File tree

5 files changed

+52
-32
lines changed

5 files changed

+52
-32
lines changed

‎drivers/iommu/dmar.c

Lines changed: 38 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1157,12 +1157,11 @@ static inline void reclaim_free_desc(struct q_inval *qi)
11571157
}
11581158
}
11591159

1160-
static int qi_check_fault(struct intel_iommu *iommu, int index)
1160+
static int qi_check_fault(struct intel_iommu *iommu, int index, int wait_index)
11611161
{
11621162
u32 fault;
11631163
int head, tail;
11641164
struct q_inval *qi = iommu->qi;
1165-
int wait_index = (index + 1) % QI_LENGTH;
11661165
int shift = qi_shift(iommu);
11671166

11681167
if (qi->desc_status[wait_index] == QI_ABORT)
@@ -1225,17 +1224,21 @@ static int qi_check_fault(struct intel_iommu *iommu, int index)
12251224
}
12261225

12271226
/*
1228-
* Submit the queued invalidation descriptor to the remapping
1229-
* hardware unit and wait for its completion.
1227+
* Function to submit invalidation descriptors of all types to the queued
1228+
* invalidation interface(QI). Multiple descriptors can be submitted at a
1229+
* time, a wait descriptor will be appended to each submission to ensure
1230+
* hardware has completed the invalidation before return. Wait descriptors
1231+
* can be part of the submission but it will not be polled for completion.
12301232
*/
1231-
int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
1233+
int qi_submit_sync(struct intel_iommu *iommu, struct qi_desc *desc,
1234+
unsigned int count, unsigned long options)
12321235
{
1233-
int rc;
12341236
struct q_inval *qi = iommu->qi;
1235-
int offset, shift, length;
12361237
struct qi_desc wait_desc;
12371238
int wait_index, index;
12381239
unsigned long flags;
1240+
int offset, shift;
1241+
int rc, i;
12391242

12401243
if (!qi)
12411244
return 0;
@@ -1244,32 +1247,41 @@ int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
12441247
rc = 0;
12451248

12461249
raw_spin_lock_irqsave(&qi->q_lock, flags);
1247-
while (qi->free_cnt < 3) {
1250+
/*
1251+
* Check if we have enough empty slots in the queue to submit,
1252+
* the calculation is based on:
1253+
* # of desc + 1 wait desc + 1 space between head and tail
1254+
*/
1255+
while (qi->free_cnt < count + 2) {
12481256
raw_spin_unlock_irqrestore(&qi->q_lock, flags);
12491257
cpu_relax();
12501258
raw_spin_lock_irqsave(&qi->q_lock, flags);
12511259
}
12521260

12531261
index = qi->free_head;
1254-
wait_index = (index + 1) % QI_LENGTH;
1262+
wait_index = (index + count) % QI_LENGTH;
12551263
shift = qi_shift(iommu);
1256-
length = 1 << shift;
12571264

1258-
qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
1265+
for (i = 0; i < count; i++) {
1266+
offset = ((index + i) % QI_LENGTH) << shift;
1267+
memcpy(qi->desc + offset, &desc[i], 1 << shift);
1268+
qi->desc_status[(index + i) % QI_LENGTH] = QI_IN_USE;
1269+
}
1270+
qi->desc_status[wait_index] = QI_IN_USE;
12591271

1260-
offset = index << shift;
1261-
memcpy(qi->desc + offset, desc, length);
12621272
wait_desc.qw0 = QI_IWD_STATUS_DATA(QI_DONE) |
12631273
QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
1274+
if (options & QI_OPT_WAIT_DRAIN)
1275+
wait_desc.qw0 |= QI_IWD_PRQ_DRAIN;
12641276
wait_desc.qw1 = virt_to_phys(&qi->desc_status[wait_index]);
12651277
wait_desc.qw2 = 0;
12661278
wait_desc.qw3 = 0;
12671279

12681280
offset = wait_index << shift;
1269-
memcpy(qi->desc + offset, &wait_desc, length);
1281+
memcpy(qi->desc + offset, &wait_desc, 1 << shift);
12701282

1271-
qi->free_head = (qi->free_head + 2) % QI_LENGTH;
1272-
qi->free_cnt -= 2;
1283+
qi->free_head = (qi->free_head + count + 1) % QI_LENGTH;
1284+
qi->free_cnt -= count + 1;
12731285

12741286
/*
12751287
* update the HW tail register indicating the presence of
@@ -1285,7 +1297,7 @@ int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
12851297
* a deadlock where the interrupt context can wait indefinitely
12861298
* for free slots in the queue.
12871299
*/
1288-
rc = qi_check_fault(iommu, index);
1300+
rc = qi_check_fault(iommu, index, wait_index);
12891301
if (rc)
12901302
break;
12911303

@@ -1294,7 +1306,8 @@ int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
12941306
raw_spin_lock(&qi->q_lock);
12951307
}
12961308

1297-
qi->desc_status[index] = QI_DONE;
1309+
for (i = 0; i < count; i++)
1310+
qi->desc_status[(index + i) % QI_LENGTH] = QI_DONE;
12981311

12991312
reclaim_free_desc(qi);
13001313
raw_spin_unlock_irqrestore(&qi->q_lock, flags);
@@ -1318,7 +1331,7 @@ void qi_global_iec(struct intel_iommu *iommu)
13181331
desc.qw3 = 0;
13191332

13201333
/* should never fail */
1321-
qi_submit_sync(&desc, iommu);
1334+
qi_submit_sync(iommu, &desc, 1, 0);
13221335
}
13231336

13241337
void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
@@ -1332,7 +1345,7 @@ void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
13321345
desc.qw2 = 0;
13331346
desc.qw3 = 0;
13341347

1335-
qi_submit_sync(&desc, iommu);
1348+
qi_submit_sync(iommu, &desc, 1, 0);
13361349
}
13371350

13381351
void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
@@ -1356,7 +1369,7 @@ void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
13561369
desc.qw2 = 0;
13571370
desc.qw3 = 0;
13581371

1359-
qi_submit_sync(&desc, iommu);
1372+
qi_submit_sync(iommu, &desc, 1, 0);
13601373
}
13611374

13621375
void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid,
@@ -1378,7 +1391,7 @@ void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid,
13781391
desc.qw2 = 0;
13791392
desc.qw3 = 0;
13801393

1381-
qi_submit_sync(&desc, iommu);
1394+
qi_submit_sync(iommu, &desc, 1, 0);
13821395
}
13831396

13841397
/* PASID-based IOTLB invalidation */
@@ -1419,7 +1432,7 @@ void qi_flush_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid, u64 addr,
14191432
QI_EIOTLB_AM(mask);
14201433
}
14211434

1422-
qi_submit_sync(&desc, iommu);
1435+
qi_submit_sync(iommu, &desc, 1, 0);
14231436
}
14241437

14251438
/* PASID-based device IOTLB Invalidate */
@@ -1448,7 +1461,7 @@ void qi_flush_dev_iotlb_pasid(struct intel_iommu *iommu, u16 sid, u16 pfsid,
14481461
if (size_order)
14491462
desc.qw1 |= QI_DEV_EIOTLB_SIZE;
14501463

1451-
qi_submit_sync(&desc, iommu);
1464+
qi_submit_sync(iommu, &desc, 1, 0);
14521465
}
14531466

14541467
void qi_flush_pasid_cache(struct intel_iommu *iommu, u16 did,
@@ -1458,7 +1471,7 @@ void qi_flush_pasid_cache(struct intel_iommu *iommu, u16 did,
14581471

14591472
desc.qw0 = QI_PC_PASID(pasid) | QI_PC_DID(did) |
14601473
QI_PC_GRAN(granu) | QI_PC_TYPE;
1461-
qi_submit_sync(&desc, iommu);
1474+
qi_submit_sync(iommu, &desc, 1, 0);
14621475
}
14631476

14641477
/*

‎drivers/iommu/intel-pasid.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -438,7 +438,7 @@ pasid_cache_invalidation_with_pasid(struct intel_iommu *iommu,
438438
desc.qw2 = 0;
439439
desc.qw3 = 0;
440440

441-
qi_submit_sync(&desc, iommu);
441+
qi_submit_sync(iommu, &desc, 1, 0);
442442
}
443443

444444
static void
@@ -452,7 +452,7 @@ iotlb_invalidation_with_pasid(struct intel_iommu *iommu, u16 did, u32 pasid)
452452
desc.qw2 = 0;
453453
desc.qw3 = 0;
454454

455-
qi_submit_sync(&desc, iommu);
455+
qi_submit_sync(iommu, &desc, 1, 0);
456456
}
457457

458458
static void

‎drivers/iommu/intel-svm.c

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -138,7 +138,7 @@ static void intel_flush_svm_range_dev (struct intel_svm *svm, struct intel_svm_d
138138
}
139139
desc.qw2 = 0;
140140
desc.qw3 = 0;
141-
qi_submit_sync(&desc, svm->iommu);
141+
qi_submit_sync(svm->iommu, &desc, 1, 0);
142142

143143
if (sdev->dev_iotlb) {
144144
desc.qw0 = QI_DEV_EIOTLB_PASID(svm->pasid) |
@@ -162,7 +162,7 @@ static void intel_flush_svm_range_dev (struct intel_svm *svm, struct intel_svm_d
162162
}
163163
desc.qw2 = 0;
164164
desc.qw3 = 0;
165-
qi_submit_sync(&desc, svm->iommu);
165+
qi_submit_sync(svm->iommu, &desc, 1, 0);
166166
}
167167
}
168168

@@ -846,7 +846,7 @@ static irqreturn_t prq_event_thread(int irq, void *d)
846846
sizeof(req->priv_data));
847847
resp.qw2 = 0;
848848
resp.qw3 = 0;
849-
qi_submit_sync(&resp, iommu);
849+
qi_submit_sync(iommu, &resp, 1, 0);
850850
}
851851
head = (head + sizeof(*req)) & PRQ_RING_MASK;
852852
}

‎drivers/iommu/intel_irq_remapping.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,7 @@ static int qi_flush_iec(struct intel_iommu *iommu, int index, int mask)
151151
desc.qw2 = 0;
152152
desc.qw3 = 0;
153153

154-
return qi_submit_sync(&desc, iommu);
154+
return qi_submit_sync(iommu, &desc, 1, 0);
155155
}
156156

157157
static int modify_irte(struct irq_2_iommu *irq_iommu,

‎include/linux/intel-iommu.h

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -333,6 +333,7 @@ enum {
333333

334334
#define QI_IWD_STATUS_DATA(d) (((u64)d) << 32)
335335
#define QI_IWD_STATUS_WRITE (((u64)1) << 5)
336+
#define QI_IWD_PRQ_DRAIN (((u64)1) << 7)
336337

337338
#define QI_IOTLB_DID(did) (((u64)did) << 16)
338339
#define QI_IOTLB_DR(dr) (((u64)dr) << 7)
@@ -702,7 +703,13 @@ void qi_flush_dev_iotlb_pasid(struct intel_iommu *iommu, u16 sid, u16 pfsid,
702703
void qi_flush_pasid_cache(struct intel_iommu *iommu, u16 did, u64 granu,
703704
int pasid);
704705

705-
extern int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu);
706+
int qi_submit_sync(struct intel_iommu *iommu, struct qi_desc *desc,
707+
unsigned int count, unsigned long options);
708+
/*
709+
* Options used in qi_submit_sync:
710+
* QI_OPT_WAIT_DRAIN - Wait for PRQ drain completion, spec 6.5.2.8.
711+
*/
712+
#define QI_OPT_WAIT_DRAIN BIT(0)
706713

707714
extern int dmar_ir_support(void);
708715

0 commit comments

Comments
 (0)