Skip to content

Commit 79b98ed

Browse files
torvaldstorvalds
committed
Merge tag 'erofs-for-6.16-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs
Pull erofs updates from Gao Xiang: "In this cycle, Intel QAT hardware accelerators are supported to improve DEFLATE decompression performance. I've tested it with the enwik9 dataset of 1 MiB pclusters on our Intel Sapphire Rapids bare-metal server and a PL0 ESSD, and the sequential read performance even surpasses LZ4 software decompression on this setup. In addition, a `fsoffset` mount option is introduced for file-backed mounts to specify the filesystem offset in order to adapt customized container formats. And other improvements and minor cleanups. Summary: - Add a `fsoffset` mount option to specify the filesystem offset - Support Intel QAT accelerators to boost up the DEFLATE algorithm - Initialize per-CPU workers and CPU hotplug hooks lazily to avoid unnecessary overhead when EROFS is not mounted - Fix file handle encoding for 64-bit NIDs - Minor cleanups" * tag 'erofs-for-6.16-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs: erofs: support DEFLATE decompression by using Intel QAT erofs: clean up erofs_{init,exit}_sysfs() erofs: add 'fsoffset' mount option to specify filesystem offset erofs: lazily initialize per-CPU workers and CPU hotplug hooks erofs: refine readahead tracepoint erofs: avoid using multiple devices with different type erofs: fix file handle encoding for 64-bit NIDs
2 parents 522544f + b4a29ef commit 79b98ed

File tree

14 files changed

+397
-64
lines changed

14 files changed

+397
-64
lines changed

‎Documentation/ABI/testing/sysfs-fs-erofs

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,3 +27,11 @@ Description: Writing to this will drop compression-related caches,
2727
- 1 : invalidate cached compressed folios
2828
- 2 : drop in-memory pclusters
2929
- 3 : drop in-memory pclusters and cached compressed folios
30+
31+
What: /sys/fs/erofs/accel
32+
Date: May 2025
33+
Contact: "Bo Liu" <[email protected]>
34+
Description: Used to set or show hardware accelerators in effect
35+
and multiple accelerators are separated by '\n'.
36+
Supported accelerator(s): qat_deflate.
37+
Disable all accelerators with an empty string (echo > accel).

‎Documentation/filesystems/erofs.rst

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,7 @@ device=%s Specify a path to an extra device to be used together.
128128
fsid=%s Specify a filesystem image ID for Fscache back-end.
129129
domain_id=%s Specify a domain ID in fscache mode so that different images
130130
with the same blobs under a given domain ID can share storage.
131+
fsoffset=%llu Specify block-aligned filesystem offset for the primary device.
131132
=================== =========================================================
132133

133134
Sysfs Entries

‎fs/erofs/Kconfig

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,20 @@ config EROFS_FS_ZIP_ZSTD
144144

145145
If unsure, say N.
146146

147+
config EROFS_FS_ZIP_ACCEL
148+
bool "EROFS hardware decompression support"
149+
depends on EROFS_FS_ZIP
150+
help
151+
Saying Y here includes hardware accelerator support for reading
152+
EROFS file systems containing compressed data. It gives better
153+
decompression speed than the software-implemented decompression, and
154+
it costs lower CPU overhead.
155+
156+
Hardware accelerator support is an experimental feature for now and
157+
file systems are still readable without selecting this option.
158+
159+
If unsure, say N.
160+
147161
config EROFS_FS_ONDEMAND
148162
bool "EROFS fscache-based on-demand read support (deprecated)"
149163
depends on EROFS_FS

‎fs/erofs/Makefile

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,5 +7,6 @@ erofs-$(CONFIG_EROFS_FS_ZIP) += decompressor.o zmap.o zdata.o zutil.o
77
erofs-$(CONFIG_EROFS_FS_ZIP_LZMA) += decompressor_lzma.o
88
erofs-$(CONFIG_EROFS_FS_ZIP_DEFLATE) += decompressor_deflate.o
99
erofs-$(CONFIG_EROFS_FS_ZIP_ZSTD) += decompressor_zstd.o
10+
erofs-$(CONFIG_EROFS_FS_ZIP_ACCEL) += decompressor_crypto.o
1011
erofs-$(CONFIG_EROFS_FS_BACKED_BY_FILE) += fileio.o
1112
erofs-$(CONFIG_EROFS_FS_ONDEMAND) += fscache.o

‎fs/erofs/compress.h

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,4 +76,14 @@ int z_erofs_fixup_insize(struct z_erofs_decompress_req *rq, const char *padbuf,
7676
unsigned int padbufsize);
7777
int __init z_erofs_init_decompressor(void);
7878
void z_erofs_exit_decompressor(void);
79+
int z_erofs_crypto_decompress(struct z_erofs_decompress_req *rq,
80+
struct page **pgpl);
81+
int z_erofs_crypto_enable_engine(const char *name, int len);
82+
#ifdef CONFIG_EROFS_FS_ZIP_ACCEL
83+
void z_erofs_crypto_disable_all_engines(void);
84+
int z_erofs_crypto_show_engines(char *buf, int size, char sep);
85+
#else
86+
static inline void z_erofs_crypto_disable_all_engines(void) {}
87+
static inline int z_erofs_crypto_show_engines(char *buf, int size, char sep) { return 0; }
88+
#endif
7989
#endif

‎fs/erofs/data.c

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ void erofs_put_metabuf(struct erofs_buf *buf)
2727

2828
void *erofs_bread(struct erofs_buf *buf, erofs_off_t offset, bool need_kmap)
2929
{
30-
pgoff_t index = offset >> PAGE_SHIFT;
30+
pgoff_t index = (buf->off + offset) >> PAGE_SHIFT;
3131
struct folio *folio = NULL;
3232

3333
if (buf->page) {
@@ -54,6 +54,7 @@ void erofs_init_metabuf(struct erofs_buf *buf, struct super_block *sb)
5454
struct erofs_sb_info *sbi = EROFS_SB(sb);
5555

5656
buf->file = NULL;
57+
buf->off = sbi->dif0.fsoff;
5758
if (erofs_is_fileio_mode(sbi)) {
5859
buf->file = sbi->dif0.file; /* some fs like FUSE needs it */
5960
buf->mapping = buf->file->f_mapping;
@@ -299,7 +300,7 @@ static int erofs_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
299300
iomap->private = buf.base;
300301
} else {
301302
iomap->type = IOMAP_MAPPED;
302-
iomap->addr = mdev.m_pa;
303+
iomap->addr = mdev.m_dif->fsoff + mdev.m_pa;
303304
if (flags & IOMAP_DAX)
304305
iomap->addr += mdev.m_dif->dax_part_off;
305306
}

‎fs/erofs/decompressor_crypto.c

Lines changed: 181 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
// SPDX-License-Identifier: GPL-2.0-or-later
2+
#include <linux/scatterlist.h>
3+
#include <crypto/acompress.h>
4+
#include "compress.h"
5+
6+
static int __z_erofs_crypto_decompress(struct z_erofs_decompress_req *rq,
7+
struct crypto_acomp *tfm)
8+
{
9+
struct sg_table st_src, st_dst;
10+
struct acomp_req *req;
11+
struct crypto_wait wait;
12+
u8 *headpage;
13+
int ret;
14+
15+
headpage = kmap_local_page(*rq->in);
16+
ret = z_erofs_fixup_insize(rq, headpage + rq->pageofs_in,
17+
min_t(unsigned int, rq->inputsize,
18+
rq->sb->s_blocksize - rq->pageofs_in));
19+
kunmap_local(headpage);
20+
if (ret)
21+
return ret;
22+
23+
req = acomp_request_alloc(tfm);
24+
if (!req)
25+
return -ENOMEM;
26+
27+
ret = sg_alloc_table_from_pages_segment(&st_src, rq->in, rq->inpages,
28+
rq->pageofs_in, rq->inputsize, UINT_MAX, GFP_KERNEL);
29+
if (ret < 0)
30+
goto failed_src_alloc;
31+
32+
ret = sg_alloc_table_from_pages_segment(&st_dst, rq->out, rq->outpages,
33+
rq->pageofs_out, rq->outputsize, UINT_MAX, GFP_KERNEL);
34+
if (ret < 0)
35+
goto failed_dst_alloc;
36+
37+
acomp_request_set_params(req, st_src.sgl,
38+
st_dst.sgl, rq->inputsize, rq->outputsize);
39+
40+
crypto_init_wait(&wait);
41+
acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
42+
crypto_req_done, &wait);
43+
44+
ret = crypto_wait_req(crypto_acomp_decompress(req), &wait);
45+
if (ret) {
46+
erofs_err(rq->sb, "failed to decompress %d in[%u, %u] out[%u]",
47+
ret, rq->inputsize, rq->pageofs_in, rq->outputsize);
48+
ret = -EIO;
49+
}
50+
51+
sg_free_table(&st_dst);
52+
failed_dst_alloc:
53+
sg_free_table(&st_src);
54+
failed_src_alloc:
55+
acomp_request_free(req);
56+
return ret;
57+
}
58+
59+
struct z_erofs_crypto_engine {
60+
char *crypto_name;
61+
struct crypto_acomp *tfm;
62+
};
63+
64+
struct z_erofs_crypto_engine *z_erofs_crypto[Z_EROFS_COMPRESSION_MAX] = {
65+
[Z_EROFS_COMPRESSION_LZ4] = (struct z_erofs_crypto_engine[]) {
66+
{},
67+
},
68+
[Z_EROFS_COMPRESSION_LZMA] = (struct z_erofs_crypto_engine[]) {
69+
{},
70+
},
71+
[Z_EROFS_COMPRESSION_DEFLATE] = (struct z_erofs_crypto_engine[]) {
72+
{ .crypto_name = "qat_deflate", },
73+
{},
74+
},
75+
[Z_EROFS_COMPRESSION_ZSTD] = (struct z_erofs_crypto_engine[]) {
76+
{},
77+
},
78+
};
79+
static DECLARE_RWSEM(z_erofs_crypto_rwsem);
80+
81+
static struct crypto_acomp *z_erofs_crypto_get_engine(int alg)
82+
{
83+
struct z_erofs_crypto_engine *e;
84+
85+
for (e = z_erofs_crypto[alg]; e->crypto_name; ++e)
86+
if (e->tfm)
87+
return e->tfm;
88+
return NULL;
89+
}
90+
91+
int z_erofs_crypto_decompress(struct z_erofs_decompress_req *rq,
92+
struct page **pgpl)
93+
{
94+
struct crypto_acomp *tfm;
95+
int i, err;
96+
97+
down_read(&z_erofs_crypto_rwsem);
98+
tfm = z_erofs_crypto_get_engine(rq->alg);
99+
if (!tfm) {
100+
err = -EOPNOTSUPP;
101+
goto out;
102+
}
103+
104+
for (i = 0; i < rq->outpages; i++) {
105+
struct page *const page = rq->out[i];
106+
struct page *victim;
107+
108+
if (!page) {
109+
victim = __erofs_allocpage(pgpl, rq->gfp, true);
110+
if (!victim) {
111+
err = -ENOMEM;
112+
goto out;
113+
}
114+
set_page_private(victim, Z_EROFS_SHORTLIVED_PAGE);
115+
rq->out[i] = victim;
116+
}
117+
}
118+
err = __z_erofs_crypto_decompress(rq, tfm);
119+
out:
120+
up_read(&z_erofs_crypto_rwsem);
121+
return err;
122+
}
123+
124+
int z_erofs_crypto_enable_engine(const char *name, int len)
125+
{
126+
struct z_erofs_crypto_engine *e;
127+
struct crypto_acomp *tfm;
128+
int alg;
129+
130+
down_write(&z_erofs_crypto_rwsem);
131+
for (alg = 0; alg < Z_EROFS_COMPRESSION_MAX; ++alg) {
132+
for (e = z_erofs_crypto[alg]; e->crypto_name; ++e) {
133+
if (!strncmp(name, e->crypto_name, len)) {
134+
if (e->tfm)
135+
break;
136+
tfm = crypto_alloc_acomp(e->crypto_name, 0, 0);
137+
if (IS_ERR(tfm)) {
138+
up_write(&z_erofs_crypto_rwsem);
139+
return -EOPNOTSUPP;
140+
}
141+
e->tfm = tfm;
142+
break;
143+
}
144+
}
145+
}
146+
up_write(&z_erofs_crypto_rwsem);
147+
return 0;
148+
}
149+
150+
void z_erofs_crypto_disable_all_engines(void)
151+
{
152+
struct z_erofs_crypto_engine *e;
153+
int alg;
154+
155+
down_write(&z_erofs_crypto_rwsem);
156+
for (alg = 0; alg < Z_EROFS_COMPRESSION_MAX; ++alg) {
157+
for (e = z_erofs_crypto[alg]; e->crypto_name; ++e) {
158+
if (!e->tfm)
159+
continue;
160+
crypto_free_acomp(e->tfm);
161+
e->tfm = NULL;
162+
}
163+
}
164+
up_write(&z_erofs_crypto_rwsem);
165+
}
166+
167+
int z_erofs_crypto_show_engines(char *buf, int size, char sep)
168+
{
169+
struct z_erofs_crypto_engine *e;
170+
int alg, len = 0;
171+
172+
for (alg = 0; alg < Z_EROFS_COMPRESSION_MAX; ++alg) {
173+
for (e = z_erofs_crypto[alg]; e->crypto_name; ++e) {
174+
if (!e->tfm)
175+
continue;
176+
len += scnprintf(buf + len, size - len, "%s%c",
177+
e->crypto_name, sep);
178+
}
179+
}
180+
return len;
181+
}

‎fs/erofs/decompressor_deflate.c

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -97,8 +97,8 @@ static int z_erofs_load_deflate_config(struct super_block *sb,
9797
return -ENOMEM;
9898
}
9999

100-
static int z_erofs_deflate_decompress(struct z_erofs_decompress_req *rq,
101-
struct page **pgpl)
100+
static int __z_erofs_deflate_decompress(struct z_erofs_decompress_req *rq,
101+
struct page **pgpl)
102102
{
103103
struct super_block *sb = rq->sb;
104104
struct z_erofs_stream_dctx dctx = { .rq = rq, .no = -1, .ni = 0 };
@@ -178,6 +178,22 @@ static int z_erofs_deflate_decompress(struct z_erofs_decompress_req *rq,
178178
return err;
179179
}
180180

181+
static int z_erofs_deflate_decompress(struct z_erofs_decompress_req *rq,
182+
struct page **pgpl)
183+
{
184+
#ifdef CONFIG_EROFS_FS_ZIP_ACCEL
185+
int err;
186+
187+
if (!rq->partial_decoding) {
188+
err = z_erofs_crypto_decompress(rq, pgpl);
189+
if (err != -EOPNOTSUPP)
190+
return err;
191+
192+
}
193+
#endif
194+
return __z_erofs_deflate_decompress(rq, pgpl);
195+
}
196+
181197
const struct z_erofs_decompressor z_erofs_deflate_decomp = {
182198
.config = z_erofs_load_deflate_config,
183199
.decompress = z_erofs_deflate_decompress,

‎fs/erofs/fileio.c

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,8 @@ static int erofs_fileio_scan_folio(struct erofs_fileio *io, struct folio *folio)
147147
if (err)
148148
break;
149149
io->rq = erofs_fileio_rq_alloc(&io->dev);
150-
io->rq->bio.bi_iter.bi_sector = io->dev.m_pa >> 9;
150+
io->rq->bio.bi_iter.bi_sector =
151+
(io->dev.m_dif->fsoff + io->dev.m_pa) >> 9;
151152
attached = 0;
152153
}
153154
if (!bio_add_folio(&io->rq->bio, folio, len, cur))
@@ -180,7 +181,7 @@ static void erofs_fileio_readahead(struct readahead_control *rac)
180181
struct folio *folio;
181182
int err;
182183

183-
trace_erofs_readpages(inode, readahead_index(rac),
184+
trace_erofs_readahead(inode, readahead_index(rac),
184185
readahead_count(rac), true);
185186
while ((folio = readahead_folio(rac))) {
186187
err = erofs_fileio_scan_folio(&io, folio);

‎fs/erofs/internal.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ struct erofs_device_info {
4444
struct erofs_fscache *fscache;
4545
struct file *file;
4646
struct dax_device *dax_dev;
47-
u64 dax_part_off;
47+
u64 fsoff, dax_part_off;
4848

4949
erofs_blk_t blocks;
5050
erofs_blk_t uniaddr;
@@ -199,6 +199,7 @@ enum {
199199
struct erofs_buf {
200200
struct address_space *mapping;
201201
struct file *file;
202+
u64 off;
202203
struct page *page;
203204
void *base;
204205
};

0 commit comments

Comments
 (0)