29
29
#define NUM_PREALLOC_POST_READ_CTXS 128
30
30
31
31
static struct kmem_cache * bio_post_read_ctx_cache ;
32
+ static struct kmem_cache * bio_entry_slab ;
32
33
static mempool_t * bio_post_read_ctx_pool ;
33
34
34
35
static bool __is_cp_guaranteed (struct page * page )
@@ -167,9 +168,10 @@ static bool f2fs_bio_post_read_required(struct bio *bio)
167
168
168
169
static void f2fs_read_end_io (struct bio * bio )
169
170
{
170
- if (time_to_inject (F2FS_P_SB (bio_first_page_all (bio )),
171
- FAULT_READ_IO )) {
172
- f2fs_show_injection_info (FAULT_READ_IO );
171
+ struct f2fs_sb_info * sbi = F2FS_P_SB (bio_first_page_all (bio ));
172
+
173
+ if (time_to_inject (sbi , FAULT_READ_IO )) {
174
+ f2fs_show_injection_info (sbi , FAULT_READ_IO );
173
175
bio -> bi_status = BLK_STS_IOERR ;
174
176
}
175
177
@@ -191,7 +193,7 @@ static void f2fs_write_end_io(struct bio *bio)
191
193
struct bvec_iter_all iter_all ;
192
194
193
195
if (time_to_inject (sbi , FAULT_WRITE_IO )) {
194
- f2fs_show_injection_info (FAULT_WRITE_IO );
196
+ f2fs_show_injection_info (sbi , FAULT_WRITE_IO );
195
197
bio -> bi_status = BLK_STS_IOERR ;
196
198
}
197
199
@@ -543,6 +545,126 @@ static bool io_is_mergeable(struct f2fs_sb_info *sbi, struct bio *bio,
543
545
return io_type_is_mergeable (io , fio );
544
546
}
545
547
548
+ static void add_bio_entry (struct f2fs_sb_info * sbi , struct bio * bio ,
549
+ struct page * page , enum temp_type temp )
550
+ {
551
+ struct f2fs_bio_info * io = sbi -> write_io [DATA ] + temp ;
552
+ struct bio_entry * be ;
553
+
554
+ be = f2fs_kmem_cache_alloc (bio_entry_slab , GFP_NOFS );
555
+ be -> bio = bio ;
556
+ bio_get (bio );
557
+
558
+ if (bio_add_page (bio , page , PAGE_SIZE , 0 ) != PAGE_SIZE )
559
+ f2fs_bug_on (sbi , 1 );
560
+
561
+ down_write (& io -> bio_list_lock );
562
+ list_add_tail (& be -> list , & io -> bio_list );
563
+ up_write (& io -> bio_list_lock );
564
+ }
565
+
566
+ static void del_bio_entry (struct bio_entry * be )
567
+ {
568
+ list_del (& be -> list );
569
+ kmem_cache_free (bio_entry_slab , be );
570
+ }
571
+
572
+ static int add_ipu_page (struct f2fs_sb_info * sbi , struct bio * * bio ,
573
+ struct page * page )
574
+ {
575
+ enum temp_type temp ;
576
+ bool found = false;
577
+ int ret = - EAGAIN ;
578
+
579
+ for (temp = HOT ; temp < NR_TEMP_TYPE && !found ; temp ++ ) {
580
+ struct f2fs_bio_info * io = sbi -> write_io [DATA ] + temp ;
581
+ struct list_head * head = & io -> bio_list ;
582
+ struct bio_entry * be ;
583
+
584
+ down_write (& io -> bio_list_lock );
585
+ list_for_each_entry (be , head , list ) {
586
+ if (be -> bio != * bio )
587
+ continue ;
588
+
589
+ found = true;
590
+
591
+ if (bio_add_page (* bio , page , PAGE_SIZE , 0 ) == PAGE_SIZE ) {
592
+ ret = 0 ;
593
+ break ;
594
+ }
595
+
596
+ /* bio is full */
597
+ del_bio_entry (be );
598
+ __submit_bio (sbi , * bio , DATA );
599
+ break ;
600
+ }
601
+ up_write (& io -> bio_list_lock );
602
+ }
603
+
604
+ if (ret ) {
605
+ bio_put (* bio );
606
+ * bio = NULL ;
607
+ }
608
+
609
+ return ret ;
610
+ }
611
+
612
+ void f2fs_submit_merged_ipu_write (struct f2fs_sb_info * sbi ,
613
+ struct bio * * bio , struct page * page )
614
+ {
615
+ enum temp_type temp ;
616
+ bool found = false;
617
+ struct bio * target = bio ? * bio : NULL ;
618
+
619
+ for (temp = HOT ; temp < NR_TEMP_TYPE && !found ; temp ++ ) {
620
+ struct f2fs_bio_info * io = sbi -> write_io [DATA ] + temp ;
621
+ struct list_head * head = & io -> bio_list ;
622
+ struct bio_entry * be ;
623
+
624
+ if (list_empty (head ))
625
+ continue ;
626
+
627
+ down_read (& io -> bio_list_lock );
628
+ list_for_each_entry (be , head , list ) {
629
+ if (target )
630
+ found = (target == be -> bio );
631
+ else
632
+ found = __has_merged_page (be -> bio , NULL ,
633
+ page , 0 );
634
+ if (found )
635
+ break ;
636
+ }
637
+ up_read (& io -> bio_list_lock );
638
+
639
+ if (!found )
640
+ continue ;
641
+
642
+ found = false;
643
+
644
+ down_write (& io -> bio_list_lock );
645
+ list_for_each_entry (be , head , list ) {
646
+ if (target )
647
+ found = (target == be -> bio );
648
+ else
649
+ found = __has_merged_page (be -> bio , NULL ,
650
+ page , 0 );
651
+ if (found ) {
652
+ target = be -> bio ;
653
+ del_bio_entry (be );
654
+ break ;
655
+ }
656
+ }
657
+ up_write (& io -> bio_list_lock );
658
+ }
659
+
660
+ if (found )
661
+ __submit_bio (sbi , target , DATA );
662
+ if (bio && * bio ) {
663
+ bio_put (* bio );
664
+ * bio = NULL ;
665
+ }
666
+ }
667
+
546
668
int f2fs_merge_page_bio (struct f2fs_io_info * fio )
547
669
{
548
670
struct bio * bio = * fio -> bio ;
@@ -557,20 +679,17 @@ int f2fs_merge_page_bio(struct f2fs_io_info *fio)
557
679
f2fs_trace_ios (fio , 0 );
558
680
559
681
if (bio && !page_is_mergeable (fio -> sbi , bio , * fio -> last_block ,
560
- fio -> new_blkaddr )) {
561
- __submit_bio (fio -> sbi , bio , fio -> type );
562
- bio = NULL ;
563
- }
682
+ fio -> new_blkaddr ))
683
+ f2fs_submit_merged_ipu_write (fio -> sbi , & bio , NULL );
564
684
alloc_new :
565
685
if (!bio ) {
566
686
bio = __bio_alloc (fio , BIO_MAX_PAGES );
567
687
bio_set_op_attrs (bio , fio -> op , fio -> op_flags );
568
- }
569
688
570
- if ( bio_add_page ( bio , page , PAGE_SIZE , 0 ) < PAGE_SIZE ) {
571
- __submit_bio ( fio -> sbi , bio , fio -> type );
572
- bio = NULL ;
573
- goto alloc_new ;
689
+ add_bio_entry ( fio -> sbi , bio , page , fio -> temp );
690
+ } else {
691
+ if ( add_ipu_page ( fio -> sbi , & bio , page ))
692
+ goto alloc_new ;
574
693
}
575
694
576
695
if (fio -> io_wbc )
@@ -584,19 +703,6 @@ int f2fs_merge_page_bio(struct f2fs_io_info *fio)
584
703
return 0 ;
585
704
}
586
705
587
- static void f2fs_submit_ipu_bio (struct f2fs_sb_info * sbi , struct bio * * bio ,
588
- struct page * page )
589
- {
590
- if (!bio )
591
- return ;
592
-
593
- if (!__has_merged_page (* bio , NULL , page , 0 ))
594
- return ;
595
-
596
- __submit_bio (sbi , * bio , DATA );
597
- * bio = NULL ;
598
- }
599
-
600
706
void f2fs_submit_page_write (struct f2fs_io_info * fio )
601
707
{
602
708
struct f2fs_sb_info * sbi = fio -> sbi ;
@@ -2098,7 +2204,7 @@ static int __write_data_page(struct page *page, bool *submitted,
2098
2204
loff_t i_size = i_size_read (inode );
2099
2205
const pgoff_t end_index = ((unsigned long long ) i_size )
2100
2206
>> PAGE_SHIFT ;
2101
- loff_t psize = (page -> index + 1 ) << PAGE_SHIFT ;
2207
+ loff_t psize = (loff_t )( page -> index + 1 ) << PAGE_SHIFT ;
2102
2208
unsigned offset = 0 ;
2103
2209
bool need_balance_fs = false;
2104
2210
int err = 0 ;
@@ -2215,14 +2321,12 @@ static int __write_data_page(struct page *page, bool *submitted,
2215
2321
2216
2322
unlock_page (page );
2217
2323
if (!S_ISDIR (inode -> i_mode ) && !IS_NOQUOTA (inode ) &&
2218
- !F2FS_I (inode )-> cp_task ) {
2219
- f2fs_submit_ipu_bio (sbi , bio , page );
2324
+ !F2FS_I (inode )-> cp_task )
2220
2325
f2fs_balance_fs (sbi , need_balance_fs );
2221
- }
2222
2326
2223
2327
if (unlikely (f2fs_cp_error (sbi ))) {
2224
- f2fs_submit_ipu_bio (sbi , bio , page );
2225
2328
f2fs_submit_merged_write (sbi , DATA );
2329
+ f2fs_submit_merged_ipu_write (sbi , bio , NULL );
2226
2330
submitted = NULL ;
2227
2331
}
2228
2332
@@ -2342,13 +2446,11 @@ static int f2fs_write_cache_pages(struct address_space *mapping,
2342
2446
}
2343
2447
2344
2448
if (PageWriteback (page )) {
2345
- if (wbc -> sync_mode != WB_SYNC_NONE ) {
2449
+ if (wbc -> sync_mode != WB_SYNC_NONE )
2346
2450
f2fs_wait_on_page_writeback (page ,
2347
2451
DATA , true, true);
2348
- f2fs_submit_ipu_bio (sbi , & bio , page );
2349
- } else {
2452
+ else
2350
2453
goto continue_unlock ;
2351
- }
2352
2454
}
2353
2455
2354
2456
if (!clear_page_dirty_for_io (page ))
@@ -2406,7 +2508,7 @@ static int f2fs_write_cache_pages(struct address_space *mapping,
2406
2508
NULL , 0 , DATA );
2407
2509
/* submit cached bio of IPU write */
2408
2510
if (bio )
2409
- __submit_bio (sbi , bio , DATA );
2511
+ f2fs_submit_merged_ipu_write (sbi , & bio , NULL );
2410
2512
2411
2513
return ret ;
2412
2514
}
@@ -3211,8 +3313,22 @@ int __init f2fs_init_post_read_processing(void)
3211
3313
return - ENOMEM ;
3212
3314
}
3213
3315
3214
- void __exit f2fs_destroy_post_read_processing (void )
3316
+ void f2fs_destroy_post_read_processing (void )
3215
3317
{
3216
3318
mempool_destroy (bio_post_read_ctx_pool );
3217
3319
kmem_cache_destroy (bio_post_read_ctx_cache );
3218
3320
}
3321
+
3322
+ int __init f2fs_init_bio_entry_cache (void )
3323
+ {
3324
+ bio_entry_slab = f2fs_kmem_cache_create ("bio_entry_slab" ,
3325
+ sizeof (struct bio_entry ));
3326
+ if (!bio_entry_slab )
3327
+ return - ENOMEM ;
3328
+ return 0 ;
3329
+ }
3330
+
3331
+ void __exit f2fs_destroy_bio_entry_cache (void )
3332
+ {
3333
+ kmem_cache_destroy (bio_entry_slab );
3334
+ }
0 commit comments