Skip to content

Commit bc302d6

Browse files
Richard Changkawasaki
Richard Chang
authored and
kawasaki
committed
zram: enable asynchronous writeback
Replace the synchronous `submit_bio_wait()` with the non-blocking `submit_bio()` to perform writeback operations asynchronously. This leverages the infrastructure added in the previous commit to handle the completion of I/O operations in a dedicated kthread. This change shows 27% speed improvement of idle writeback on Android platform. Test Flow: - mkfs on the zram device, mount it - cp three linux-6.16-rc2.tar.gz tarball files as the data - do idle writeback - check bd_stat writes 185072 pages Test Result: idle writeback for 185072 4k-pages (~723 MiB) $ echo all > /sys/block/zram0/idle $ time echo idle > /sys/block/zram0/writeback Async writeback: 0m02.49s real 0m00.00s user 0m01.19s system 0m02.32s real 0m00.00s user 0m00.89s system 0m02.35s real 0m00.00s user 0m00.93s system 0m02.29s real 0m00.00s user 0m00.88s system Sync writeback: 0m03.09s real 0m00.00s user 0m01.07s system 0m03.18s real 0m00.00s user 0m01.12s system 0m03.47s real 0m00.00s user 0m01.16s system 0m03.36s real 0m00.00s user 0m01.27s system Signed-off-by: Richard Chang <[email protected]>
1 parent 6cc3c6b commit bc302d6

File tree

4 files changed

+161
-68
lines changed

4 files changed

+161
-68
lines changed

‎drivers/block/zram/zram_drv.c‎

Lines changed: 45 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,6 @@ static size_t huge_class_size;
5757

5858
static const struct block_device_operations zram_devops;
5959

60-
static void zram_free_page(struct zram *zram, size_t index);
6160
static int zram_read_from_zspool(struct zram *zram, struct page *page,
6261
u32 index);
6362

@@ -96,7 +95,7 @@ static __must_check bool zram_slot_trylock(struct zram *zram, u32 index)
9695
return false;
9796
}
9897

99-
static void zram_slot_lock(struct zram *zram, u32 index)
98+
void zram_slot_lock(struct zram *zram, u32 index)
10099
{
101100
unsigned long *lock = &zram->table[index].flags;
102101

@@ -105,7 +104,7 @@ static void zram_slot_lock(struct zram *zram, u32 index)
105104
lock_acquired(slot_dep_map(zram, index), _RET_IP_);
106105
}
107106

108-
static void zram_slot_unlock(struct zram *zram, u32 index)
107+
void zram_slot_unlock(struct zram *zram, u32 index)
109108
{
110109
unsigned long *lock = &zram->table[index].flags;
111110

@@ -128,19 +127,17 @@ static unsigned long zram_get_handle(struct zram *zram, u32 index)
128127
return zram->table[index].handle;
129128
}
130129

131-
static void zram_set_handle(struct zram *zram, u32 index, unsigned long handle)
130+
void zram_set_handle(struct zram *zram, u32 index, unsigned long handle)
132131
{
133132
zram->table[index].handle = handle;
134133
}
135134

136-
static bool zram_test_flag(struct zram *zram, u32 index,
137-
enum zram_pageflags flag)
135+
bool zram_test_flag(struct zram *zram, u32 index, enum zram_pageflags flag)
138136
{
139137
return zram->table[index].flags & BIT(flag);
140138
}
141139

142-
static void zram_set_flag(struct zram *zram, u32 index,
143-
enum zram_pageflags flag)
140+
void zram_set_flag(struct zram *zram, u32 index, enum zram_pageflags flag)
144141
{
145142
zram->table[index].flags |= BIT(flag);
146143
}
@@ -243,22 +240,33 @@ static struct zram_pp_ctl *init_pp_ctl(void)
243240
if (!ctl)
244241
return NULL;
245242

243+
init_completion(&ctl->all_done);
244+
atomic_set(&ctl->num_pp_slots, 0);
246245
for (idx = 0; idx < NUM_PP_BUCKETS; idx++)
247246
INIT_LIST_HEAD(&ctl->pp_buckets[idx]);
248247
return ctl;
249248
}
250249

251-
static void release_pp_slot(struct zram *zram, struct zram_pp_slot *pps)
250+
static void remove_pp_slot_from_ctl(struct zram_pp_slot *pps)
252251
{
253252
list_del_init(&pps->entry);
253+
}
254254

255+
void free_pp_slot(struct zram *zram, struct zram_pp_slot *pps)
256+
{
255257
zram_slot_lock(zram, pps->index);
256258
zram_clear_flag(zram, pps->index, ZRAM_PP_SLOT);
257259
zram_slot_unlock(zram, pps->index);
258260

259261
kfree(pps);
260262
}
261263

264+
static void release_pp_slot(struct zram *zram, struct zram_pp_slot *pps)
265+
{
266+
remove_pp_slot_from_ctl(pps);
267+
free_pp_slot(zram, pps);
268+
}
269+
262270
static void release_pp_ctl(struct zram *zram, struct zram_pp_ctl *ctl)
263271
{
264272
u32 idx;
@@ -297,6 +305,7 @@ static bool place_pp_slot(struct zram *zram, struct zram_pp_ctl *ctl,
297305
list_add(&pps->entry, &ctl->pp_buckets[bid]);
298306

299307
zram_set_flag(zram, pps->index, ZRAM_PP_SLOT);
308+
atomic_inc(&ctl->num_pp_slots);
300309
return true;
301310
}
302311

@@ -697,18 +706,18 @@ static void read_from_bdev_async(struct zram *zram, struct page *page,
697706
static int zram_writeback_slots(struct zram *zram, struct zram_pp_ctl *ctl)
698707
{
699708
unsigned long blk_idx = 0;
700-
struct page *page = NULL;
701709
struct zram_pp_slot *pps;
702-
struct bio_vec bio_vec;
703-
struct bio bio;
704-
int ret = 0, err;
710+
int ret = 0;
705711
u32 index;
712+
int nr_pps = atomic_read(&ctl->num_pp_slots);
706713

707-
page = alloc_page(GFP_KERNEL);
708-
if (!page)
709-
return -ENOMEM;
714+
if (!nr_pps)
715+
return 0;
710716

711717
while ((pps = select_pp_slot(ctl))) {
718+
struct zram_wb_request *req;
719+
struct page *page;
720+
712721
spin_lock(&zram->wb_limit_lock);
713722
if (zram->wb_limit_enable && !zram->bd_wb_limit) {
714723
spin_unlock(&zram->wb_limit_lock);
@@ -725,6 +734,13 @@ static int zram_writeback_slots(struct zram *zram, struct zram_pp_ctl *ctl)
725734
}
726735
}
727736

737+
req = alloc_wb_request(zram, pps, ctl, blk_idx);
738+
if (IS_ERR(req)) {
739+
ret = PTR_ERR(req);
740+
break;
741+
}
742+
page = bio_first_page_all(req->bio);
743+
728744
index = pps->index;
729745
zram_slot_lock(zram, index);
730746
/*
@@ -739,63 +755,28 @@ static int zram_writeback_slots(struct zram *zram, struct zram_pp_ctl *ctl)
739755
goto next;
740756
zram_slot_unlock(zram, index);
741757

742-
bio_init(&bio, zram->bdev, &bio_vec, 1,
743-
REQ_OP_WRITE | REQ_SYNC);
744-
bio.bi_iter.bi_sector = blk_idx * (PAGE_SIZE >> 9);
745-
__bio_add_page(&bio, page, PAGE_SIZE, 0);
746-
747-
/*
748-
* XXX: A single page IO would be inefficient for write
749-
* but it would be not bad as starter.
750-
*/
751-
err = submit_bio_wait(&bio);
752-
if (err) {
753-
release_pp_slot(zram, pps);
754-
/*
755-
* BIO errors are not fatal, we continue and simply
756-
* attempt to writeback the remaining objects (pages).
757-
* At the same time we need to signal user-space that
758-
* some writes (at least one, but also could be all of
759-
* them) were not successful and we do so by returning
760-
* the most recent BIO error.
761-
*/
762-
ret = err;
763-
continue;
764-
}
765-
766-
atomic64_inc(&zram->stats.bd_writes);
767-
zram_slot_lock(zram, index);
768-
/*
769-
* Same as above, we release slot lock during writeback so
770-
* slot can change under us: slot_free() or slot_free() and
771-
* reallocation (zram_write_page()). In both cases slot loses
772-
* ZRAM_PP_SLOT flag. No concurrent post-processing can set
773-
* ZRAM_PP_SLOT on such slots until current post-processing
774-
* finishes.
775-
*/
776-
if (!zram_test_flag(zram, index, ZRAM_PP_SLOT))
777-
goto next;
778-
779-
zram_free_page(zram, index);
780-
zram_set_flag(zram, index, ZRAM_WB);
781-
zram_set_handle(zram, index, blk_idx);
758+
nr_pps--;
759+
remove_pp_slot_from_ctl(pps);
782760
blk_idx = 0;
783-
atomic64_inc(&zram->stats.pages_stored);
784-
spin_lock(&zram->wb_limit_lock);
785-
if (zram->wb_limit_enable && zram->bd_wb_limit > 0)
786-
zram->bd_wb_limit -= 1UL << (PAGE_SHIFT - 12);
787-
spin_unlock(&zram->wb_limit_lock);
761+
submit_bio(req->bio);
762+
continue;
763+
788764
next:
789765
zram_slot_unlock(zram, index);
790766
release_pp_slot(zram, pps);
767+
free_wb_request(req);
791768

792769
cond_resched();
793770
}
794771

795772
if (blk_idx)
796773
free_block_bdev(zram, blk_idx);
797-
if (page)
798-
__free_page(page);
774+
775+
if (nr_pps && atomic_sub_and_test(nr_pps, &ctl->num_pp_slots))
776+
complete(&ctl->all_done);
777+
778+
/* wait until all async bios completed */
779+
wait_for_completion(&ctl->all_done);
799780

800781
return ret;
801782
}
@@ -1579,7 +1560,7 @@ static bool zram_meta_alloc(struct zram *zram, u64 disksize)
15791560
return true;
15801561
}
15811562

1582-
static void zram_free_page(struct zram *zram, size_t index)
1563+
void zram_free_page(struct zram *zram, size_t index)
15831564
{
15841565
unsigned long handle;
15851566

‎drivers/block/zram/zram_drv.h‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,13 @@ struct zram {
140140
atomic_t pp_in_progress;
141141
};
142142

143+
void zram_slot_lock(struct zram *zram, u32 index);
144+
void zram_slot_unlock(struct zram *zram, u32 index);
145+
void zram_set_handle(struct zram *zram, u32 index, unsigned long handle);
146+
bool zram_test_flag(struct zram *zram, u32 index, enum zram_pageflags flag);
147+
void zram_set_flag(struct zram *zram, u32 index, enum zram_pageflags flag);
148+
void zram_free_page(struct zram *zram, size_t index);
149+
143150
#if defined CONFIG_ZRAM_WRITEBACK || defined CONFIG_ZRAM_MULTI_COMP
144151
struct zram_pp_slot {
145152
unsigned long index;
@@ -155,7 +162,11 @@ struct zram_pp_slot {
155162

156163
struct zram_pp_ctl {
157164
struct list_head pp_buckets[NUM_PP_BUCKETS];
165+
struct completion all_done;
166+
atomic_t num_pp_slots;
158167
};
168+
169+
void free_pp_slot(struct zram *zram, struct zram_pp_slot *pps);
159170
#endif
160171

161172
#endif

‎drivers/block/zram/zram_wb.c‎

Lines changed: 101 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,15 +43,60 @@ void free_block_bdev(struct zram *zram, unsigned long blk_idx)
4343
static void complete_wb_request(struct zram_wb_request *req)
4444
{
4545
struct zram *zram = req->zram;
46+
struct zram_pp_slot *pps = req->pps;
47+
struct zram_pp_ctl *ctl = req->ppctl;
48+
unsigned long index = pps->index;
4649
unsigned long blk_idx = req->blk_idx;
50+
struct bio *bio = req->bio;
51+
52+
if (bio->bi_status)
53+
goto out_err;
54+
55+
atomic64_inc(&zram->stats.bd_writes);
56+
zram_slot_lock(zram, index);
57+
58+
/*
59+
* We release slot lock during writeback so slot can change
60+
* under us: slot_free() or slot_free() and reallocation
61+
* (zram_write_page()). In both cases slot loses
62+
* ZRAM_PP_SLOT flag. No concurrent post-processing can set
63+
* ZRAM_PP_SLOT on such slots until current post-processing
64+
* finishes.
65+
*/
66+
if (!zram_test_flag(zram, index, ZRAM_PP_SLOT)) {
67+
zram_slot_unlock(zram, index);
68+
goto out_err;
69+
}
4770

71+
zram_free_page(zram, index);
72+
zram_set_flag(zram, index, ZRAM_WB);
73+
zram_set_handle(zram, index, blk_idx);
74+
atomic64_inc(&zram->stats.pages_stored);
75+
spin_lock(&zram->wb_limit_lock);
76+
if (zram->wb_limit_enable && zram->bd_wb_limit > 0)
77+
zram->bd_wb_limit -= 1UL << (PAGE_SHIFT - 12);
78+
spin_unlock(&zram->wb_limit_lock);
79+
zram_slot_unlock(zram, index);
80+
goto end;
81+
82+
out_err:
4883
free_block_bdev(zram, blk_idx);
84+
end:
85+
free_pp_slot(zram, pps);
4986
free_wb_request(req);
87+
88+
if (atomic_dec_and_test(&ctl->num_pp_slots))
89+
complete(&ctl->all_done);
5090
}
5191

52-
void enqueue_wb_request(struct zram_wb_request_list *req_list,
53-
struct zram_wb_request *req)
92+
static void enqueue_wb_request(struct zram_wb_request_list *req_list,
93+
struct zram_wb_request *req)
5494
{
95+
/*
96+
* The enqueue path comes from softirq context:
97+
* blk_done_softirq -> bio_endio -> zram_writeback_end_io
98+
* Use spin_lock_bh for locking.
99+
*/
55100
spin_lock_bh(&req_list->lock);
56101
list_add_tail(&req->node, &req_list->head);
57102
req_list->count++;
@@ -83,6 +128,7 @@ static void destroy_wb_request_list(struct zram_wb_request_list *req_list)
83128
while (!list_empty(&req_list->head)) {
84129
req = dequeue_wb_request(req_list);
85130
free_block_bdev(req->zram, req->blk_idx);
131+
free_pp_slot(req->zram, req->pps);
86132
free_wb_request(req);
87133
}
88134
}
@@ -117,6 +163,59 @@ static int wb_thread_func(void *data)
117163
return 0;
118164
}
119165

166+
static void zram_writeback_end_io(struct bio *bio)
167+
{
168+
struct zram_wb_request *req =
169+
(struct zram_wb_request *)bio->bi_private;
170+
171+
enqueue_wb_request(&wb_req_list, req);
172+
wake_up(&wb_wq);
173+
}
174+
175+
struct zram_wb_request *alloc_wb_request(struct zram *zram,
176+
struct zram_pp_slot *pps,
177+
struct zram_pp_ctl *ppctl,
178+
unsigned long blk_idx)
179+
{
180+
struct zram_wb_request *req;
181+
struct page *page;
182+
struct bio *bio;
183+
int err = 0;
184+
185+
page = alloc_page(GFP_NOIO | __GFP_NOWARN);
186+
if (!page)
187+
return ERR_PTR(-ENOMEM);
188+
189+
bio = bio_alloc(zram->bdev, 1, REQ_OP_WRITE, GFP_NOIO | __GFP_NOWARN);
190+
if (!bio) {
191+
err = -ENOMEM;
192+
goto out_free_page;
193+
}
194+
195+
req = kmalloc(sizeof(struct zram_wb_request), GFP_NOIO | __GFP_NOWARN);
196+
if (!req) {
197+
err = -ENOMEM;
198+
goto out_free_bio;
199+
}
200+
req->zram = zram;
201+
req->pps = pps;
202+
req->ppctl = ppctl;
203+
req->blk_idx = blk_idx;
204+
req->bio = bio;
205+
206+
bio->bi_iter.bi_sector = blk_idx * (PAGE_SIZE >> 9);
207+
__bio_add_page(bio, page, PAGE_SIZE, 0);
208+
bio->bi_private = req;
209+
bio->bi_end_io = zram_writeback_end_io;
210+
return req;
211+
212+
out_free_bio:
213+
bio_put(bio);
214+
out_free_page:
215+
__free_page(page);
216+
return ERR_PTR(err);
217+
}
218+
120219
void free_wb_request(struct zram_wb_request *req)
121220
{
122221
struct bio *bio = req->bio;

‎drivers/block/zram/zram_wb.h‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,11 @@ unsigned long alloc_block_bdev(struct zram *zram);
2626
void free_block_bdev(struct zram *zram, unsigned long blk_idx);
2727
int setup_zram_writeback(void);
2828
void destroy_zram_writeback(void);
29+
struct zram_wb_request *alloc_wb_request(struct zram *zram,
30+
struct zram_pp_slot *pps,
31+
struct zram_pp_ctl *ppctl,
32+
unsigned long blk_idx);
2933
void free_wb_request(struct zram_wb_request *req);
30-
void enqueue_wb_request(struct zram_wb_request_list *req_list,
31-
struct zram_wb_request *req);
3234
#else
3335
inline unsigned long alloc_block_bdev(struct zram *zram) { return 0; }
3436
inline void free_block_bdev(struct zram *zram, unsigned long blk_idx) {};

0 commit comments

Comments
 (0)