Skip to content

Commit a6889be

Browse files
paulmckrcupaulmckrcu
committed
refscale: Add tests using SLAB_TYPESAFE_BY_RCU
This commit adds three read-side-only tests of three use cases featuring SLAB_TYPESAFE_BY_RCU: One using per-object reference counting, one using per-object locking, and one using per-object sequence locking. [ paulmck: Apply feedback from kernel test robot. ] Signed-off-by: Paul E. McKenney <[email protected]>
1 parent 3c6496c commit a6889be

File tree

1 file changed

+234
-0
lines changed

1 file changed

+234
-0
lines changed

‎kernel/rcu/refscale.c

Lines changed: 234 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,8 @@ torture_param(int, verbose_batched, 0, "Batch verbose debugging printk()s");
7676
// Wait until there are multiple CPUs before starting test.
7777
torture_param(int, holdoff, IS_BUILTIN(CONFIG_RCU_REF_SCALE_TEST) ? 10 : 0,
7878
"Holdoff time before test start (s)");
79+
// Number of typesafe_lookup structures, that is, the degree of concurrency.
80+
torture_param(long, lookup_instances, 0, "Number of typesafe_lookup structures.");
7981
// Number of loops per experiment, all readers execute operations concurrently.
8082
torture_param(long, loops, 10000, "Number of loops per experiment.");
8183
// Number of readers, with -1 defaulting to about 75% of the CPUs.
@@ -526,6 +528,237 @@ static struct ref_scale_ops clock_ops = {
526528
.name = "clock"
527529
};
528530

531+
////////////////////////////////////////////////////////////////////////
532+
//
533+
// Methods leveraging SLAB_TYPESAFE_BY_RCU.
534+
//
535+
536+
// Item to look up in a typesafe manner. Array of pointers to these.
537+
struct refscale_typesafe {
538+
atomic_t rts_refctr; // Used by all flavors
539+
spinlock_t rts_lock;
540+
seqlock_t rts_seqlock;
541+
unsigned int a;
542+
unsigned int b;
543+
};
544+
545+
static struct kmem_cache *typesafe_kmem_cachep;
546+
static struct refscale_typesafe **rtsarray;
547+
static long rtsarray_size;
548+
static DEFINE_TORTURE_RANDOM_PERCPU(refscale_rand);
549+
static bool (*rts_acquire)(struct refscale_typesafe *rtsp, unsigned int *start);
550+
static bool (*rts_release)(struct refscale_typesafe *rtsp, unsigned int start);
551+
552+
// Conditionally acquire an explicit in-structure reference count.
553+
static bool typesafe_ref_acquire(struct refscale_typesafe *rtsp, unsigned int *start)
554+
{
555+
return atomic_inc_not_zero(&rtsp->rts_refctr);
556+
}
557+
558+
// Unconditionally release an explicit in-structure reference count.
559+
static bool typesafe_ref_release(struct refscale_typesafe *rtsp, unsigned int start)
560+
{
561+
if (!atomic_dec_return(&rtsp->rts_refctr)) {
562+
WRITE_ONCE(rtsp->a, rtsp->a + 1);
563+
kmem_cache_free(typesafe_kmem_cachep, rtsp);
564+
}
565+
return true;
566+
}
567+
568+
// Unconditionally acquire an explicit in-structure spinlock.
569+
static bool typesafe_lock_acquire(struct refscale_typesafe *rtsp, unsigned int *start)
570+
{
571+
spin_lock(&rtsp->rts_lock);
572+
return true;
573+
}
574+
575+
// Unconditionally release an explicit in-structure spinlock.
576+
static bool typesafe_lock_release(struct refscale_typesafe *rtsp, unsigned int start)
577+
{
578+
spin_unlock(&rtsp->rts_lock);
579+
return true;
580+
}
581+
582+
// Unconditionally acquire an explicit in-structure sequence lock.
583+
static bool typesafe_seqlock_acquire(struct refscale_typesafe *rtsp, unsigned int *start)
584+
{
585+
*start = read_seqbegin(&rtsp->rts_seqlock);
586+
return true;
587+
}
588+
589+
// Conditionally release an explicit in-structure sequence lock. Return
590+
// true if this release was successful, that is, if no retry is required.
591+
static bool typesafe_seqlock_release(struct refscale_typesafe *rtsp, unsigned int start)
592+
{
593+
return !read_seqretry(&rtsp->rts_seqlock, start);
594+
}
595+
596+
// Do a read-side critical section with the specified delay in
597+
// microseconds and nanoseconds inserted so as to increase probability
598+
// of failure.
599+
static void typesafe_delay_section(const int nloops, const int udl, const int ndl)
600+
{
601+
unsigned int a;
602+
unsigned int b;
603+
int i;
604+
long idx;
605+
struct refscale_typesafe *rtsp;
606+
unsigned int start;
607+
608+
for (i = nloops; i >= 0; i--) {
609+
preempt_disable();
610+
idx = torture_random(this_cpu_ptr(&refscale_rand)) % rtsarray_size;
611+
preempt_enable();
612+
retry:
613+
rcu_read_lock();
614+
rtsp = rcu_dereference(rtsarray[idx]);
615+
a = READ_ONCE(rtsp->a);
616+
if (!rts_acquire(rtsp, &start)) {
617+
rcu_read_unlock();
618+
goto retry;
619+
}
620+
if (a != READ_ONCE(rtsp->a)) {
621+
(void)rts_release(rtsp, start);
622+
rcu_read_unlock();
623+
goto retry;
624+
}
625+
un_delay(udl, ndl);
626+
// Remember, seqlock read-side release can fail.
627+
if (!rts_release(rtsp, start)) {
628+
rcu_read_unlock();
629+
goto retry;
630+
}
631+
b = READ_ONCE(rtsp->a);
632+
WARN_ONCE(a != b, "Re-read of ->a changed from %u to %u.\n", a, b);
633+
b = rtsp->b;
634+
rcu_read_unlock();
635+
WARN_ON_ONCE(a * a != b);
636+
}
637+
}
638+
639+
// Because the acquisition and release methods are expensive, there
640+
// is no point in optimizing away the un_delay() function's two checks.
641+
// Thus simply define typesafe_read_section() as a simple wrapper around
642+
// typesafe_delay_section().
643+
static void typesafe_read_section(const int nloops)
644+
{
645+
typesafe_delay_section(nloops, 0, 0);
646+
}
647+
648+
// Allocate and initialize one refscale_typesafe structure.
649+
static struct refscale_typesafe *typesafe_alloc_one(void)
650+
{
651+
struct refscale_typesafe *rtsp;
652+
653+
rtsp = kmem_cache_alloc(typesafe_kmem_cachep, GFP_KERNEL);
654+
if (!rtsp)
655+
return NULL;
656+
atomic_set(&rtsp->rts_refctr, 1);
657+
WRITE_ONCE(rtsp->a, rtsp->a + 1);
658+
WRITE_ONCE(rtsp->b, rtsp->a * rtsp->a);
659+
return rtsp;
660+
}
661+
662+
// Slab-allocator constructor for refscale_typesafe structures created
663+
// out of a new slab of system memory.
664+
static void refscale_typesafe_ctor(void *rtsp_in)
665+
{
666+
struct refscale_typesafe *rtsp = rtsp_in;
667+
668+
spin_lock_init(&rtsp->rts_lock);
669+
seqlock_init(&rtsp->rts_seqlock);
670+
preempt_disable();
671+
rtsp->a = torture_random(this_cpu_ptr(&refscale_rand));
672+
preempt_enable();
673+
}
674+
675+
static struct ref_scale_ops typesafe_ref_ops;
676+
static struct ref_scale_ops typesafe_lock_ops;
677+
static struct ref_scale_ops typesafe_seqlock_ops;
678+
679+
// Initialize for a typesafe test.
680+
static bool typesafe_init(void)
681+
{
682+
long idx;
683+
long si = lookup_instances;
684+
685+
typesafe_kmem_cachep = kmem_cache_create("refscale_typesafe",
686+
sizeof(struct refscale_typesafe), sizeof(void *),
687+
SLAB_TYPESAFE_BY_RCU, refscale_typesafe_ctor);
688+
if (!typesafe_kmem_cachep)
689+
return false;
690+
if (si < 0)
691+
si = -si * nr_cpu_ids;
692+
else if (si == 0)
693+
si = nr_cpu_ids;
694+
rtsarray_size = si;
695+
rtsarray = kcalloc(si, sizeof(*rtsarray), GFP_KERNEL);
696+
if (!rtsarray)
697+
return false;
698+
for (idx = 0; idx < rtsarray_size; idx++) {
699+
rtsarray[idx] = typesafe_alloc_one();
700+
if (!rtsarray[idx])
701+
return false;
702+
}
703+
if (cur_ops == &typesafe_ref_ops) {
704+
rts_acquire = typesafe_ref_acquire;
705+
rts_release = typesafe_ref_release;
706+
} else if (cur_ops == &typesafe_lock_ops) {
707+
rts_acquire = typesafe_lock_acquire;
708+
rts_release = typesafe_lock_release;
709+
} else if (cur_ops == &typesafe_seqlock_ops) {
710+
rts_acquire = typesafe_seqlock_acquire;
711+
rts_release = typesafe_seqlock_release;
712+
} else {
713+
WARN_ON_ONCE(1);
714+
return false;
715+
}
716+
return true;
717+
}
718+
719+
// Clean up after a typesafe test.
720+
static void typesafe_cleanup(void)
721+
{
722+
long idx;
723+
724+
if (rtsarray) {
725+
for (idx = 0; idx < rtsarray_size; idx++)
726+
kmem_cache_free(typesafe_kmem_cachep, rtsarray[idx]);
727+
kfree(rtsarray);
728+
rtsarray = NULL;
729+
rtsarray_size = 0;
730+
}
731+
kmem_cache_destroy(typesafe_kmem_cachep);
732+
typesafe_kmem_cachep = NULL;
733+
rts_acquire = NULL;
734+
rts_release = NULL;
735+
}
736+
737+
// The typesafe_init() function distinguishes these structures by address.
738+
static struct ref_scale_ops typesafe_ref_ops = {
739+
.init = typesafe_init,
740+
.cleanup = typesafe_cleanup,
741+
.readsection = typesafe_read_section,
742+
.delaysection = typesafe_delay_section,
743+
.name = "typesafe_ref"
744+
};
745+
746+
static struct ref_scale_ops typesafe_lock_ops = {
747+
.init = typesafe_init,
748+
.cleanup = typesafe_cleanup,
749+
.readsection = typesafe_read_section,
750+
.delaysection = typesafe_delay_section,
751+
.name = "typesafe_lock"
752+
};
753+
754+
static struct ref_scale_ops typesafe_seqlock_ops = {
755+
.init = typesafe_init,
756+
.cleanup = typesafe_cleanup,
757+
.readsection = typesafe_read_section,
758+
.delaysection = typesafe_delay_section,
759+
.name = "typesafe_seqlock"
760+
};
761+
529762
static void rcu_scale_one_reader(void)
530763
{
531764
if (readdelay <= 0)
@@ -815,6 +1048,7 @@ ref_scale_init(void)
8151048
static struct ref_scale_ops *scale_ops[] = {
8161049
&rcu_ops, &srcu_ops, RCU_TRACE_OPS RCU_TASKS_OPS &refcnt_ops, &rwlock_ops,
8171050
&rwsem_ops, &lock_ops, &lock_irq_ops, &acqrel_ops, &clock_ops,
1051+
&typesafe_ref_ops, &typesafe_lock_ops, &typesafe_seqlock_ops,
8181052
};
8191053

8201054
if (!torture_init_begin(scale_type, verbose))

0 commit comments

Comments
 (0)