@@ -1319,22 +1319,23 @@ struct bp_patching_desc {
1319
1319
atomic_t refs ;
1320
1320
};
1321
1321
1322
- static struct bp_patching_desc * bp_desc ;
1322
+ static struct bp_patching_desc bp_desc ;
1323
1323
1324
1324
static __always_inline
1325
- struct bp_patching_desc * try_get_desc (struct bp_patching_desc * * descp )
1325
+ struct bp_patching_desc * try_get_desc (void )
1326
1326
{
1327
- /* rcu_dereference */
1328
- struct bp_patching_desc * desc = __READ_ONCE (* descp );
1327
+ struct bp_patching_desc * desc = & bp_desc ;
1329
1328
1330
- if (!desc || ! arch_atomic_inc_not_zero (& desc -> refs ))
1329
+ if (!arch_atomic_inc_not_zero (& desc -> refs ))
1331
1330
return NULL ;
1332
1331
1333
1332
return desc ;
1334
1333
}
1335
1334
1336
- static __always_inline void put_desc (struct bp_patching_desc * desc )
1335
+ static __always_inline void put_desc (void )
1337
1336
{
1337
+ struct bp_patching_desc * desc = & bp_desc ;
1338
+
1338
1339
smp_mb__before_atomic ();
1339
1340
arch_atomic_dec (& desc -> refs );
1340
1341
}
@@ -1367,15 +1368,15 @@ noinstr int poke_int3_handler(struct pt_regs *regs)
1367
1368
1368
1369
/*
1369
1370
* Having observed our INT3 instruction, we now must observe
1370
- * bp_desc:
1371
+ * bp_desc with non-zero refcount :
1371
1372
*
1372
- * bp_desc = desc INT3
1373
+ * bp_desc.refs = 1 INT3
1373
1374
* WMB RMB
1374
- * write INT3 if (desc )
1375
+ * write INT3 if (bp_desc.refs != 0 )
1375
1376
*/
1376
1377
smp_rmb ();
1377
1378
1378
- desc = try_get_desc (& bp_desc );
1379
+ desc = try_get_desc ();
1379
1380
if (!desc )
1380
1381
return 0 ;
1381
1382
@@ -1429,7 +1430,7 @@ noinstr int poke_int3_handler(struct pt_regs *regs)
1429
1430
ret = 1 ;
1430
1431
1431
1432
out_put :
1432
- put_desc (desc );
1433
+ put_desc ();
1433
1434
return ret ;
1434
1435
}
1435
1436
@@ -1460,18 +1461,20 @@ static int tp_vec_nr;
1460
1461
*/
1461
1462
static void text_poke_bp_batch (struct text_poke_loc * tp , unsigned int nr_entries )
1462
1463
{
1463
- struct bp_patching_desc desc = {
1464
- .vec = tp ,
1465
- .nr_entries = nr_entries ,
1466
- .refs = ATOMIC_INIT (1 ),
1467
- };
1468
1464
unsigned char int3 = INT3_INSN_OPCODE ;
1469
1465
unsigned int i ;
1470
1466
int do_sync ;
1471
1467
1472
1468
lockdep_assert_held (& text_mutex );
1473
1469
1474
- smp_store_release (& bp_desc , & desc ); /* rcu_assign_pointer */
1470
+ bp_desc .vec = tp ;
1471
+ bp_desc .nr_entries = nr_entries ;
1472
+
1473
+ /*
1474
+ * Corresponds to the implicit memory barrier in try_get_desc() to
1475
+ * ensure reading a non-zero refcount provides up to date bp_desc data.
1476
+ */
1477
+ atomic_set_release (& bp_desc .refs , 1 );
1475
1478
1476
1479
/*
1477
1480
* Corresponding read barrier in int3 notifier for making sure the
@@ -1559,12 +1562,10 @@ static void text_poke_bp_batch(struct text_poke_loc *tp, unsigned int nr_entries
1559
1562
text_poke_sync ();
1560
1563
1561
1564
/*
1562
- * Remove and synchronize_rcu(), except we have a very primitive
1563
- * refcount based completion.
1565
+ * Remove and wait for refs to be zero.
1564
1566
*/
1565
- WRITE_ONCE (bp_desc , NULL ); /* RCU_INIT_POINTER */
1566
- if (!atomic_dec_and_test (& desc .refs ))
1567
- atomic_cond_read_acquire (& desc .refs , !VAL );
1567
+ if (!atomic_dec_and_test (& bp_desc .refs ))
1568
+ atomic_cond_read_acquire (& bp_desc .refs , !VAL );
1568
1569
}
1569
1570
1570
1571
static void text_poke_loc_init (struct text_poke_loc * tp , void * addr ,
0 commit comments