Skip to content

Commit b5fe513

Browse files
imtzwimtzw
committed
rgw: fix list obj v1 nextmarker
when encoding-type=url given, aws sdk will try to decode next marker returned in a list obj response. if it is not actually encoded and the marker obj just have an encoding symbol in its name, aws sdk may get error when trying to decode it. Signed-off-by: imtzw <[email protected]>
1 parent 10d3caa commit b5fe513

File tree

2 files changed

+22
-64
lines changed

2 files changed

+22
-64
lines changed

‎src/rgw/rgw_rest.h‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -706,6 +706,16 @@ extern void dump_start(req_state *s);
706706
extern void list_all_buckets_start(req_state *s);
707707
extern void dump_owner(req_state *s, const rgw_user& id,
708708
const std::string& name, const char *section = NULL);
709+
inline void dump_urlsafe(req_state *s, bool encode_key, const char* key, const std::string& val, bool encode_slash = true) {
710+
if (encode_key) {
711+
std::string _val;
712+
url_encode(val, _val, encode_slash);
713+
s->formatter->dump_string(key, _val);
714+
}
715+
else {
716+
s->formatter->dump_string(key, val);
717+
}
718+
}
709719
extern void dump_header(req_state* s,
710720
const std::string_view& name,
711721
const std::string_view& val);

‎src/rgw/rgw_rest_s3.cc‎

Lines changed: 12 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -1707,11 +1707,7 @@ void RGWListBucket_ObjStore_S3::send_common_versioned_response()
17071707
for (pref_iter = common_prefixes.begin();
17081708
pref_iter != common_prefixes.end(); ++pref_iter) {
17091709
s->formatter->open_array_section("CommonPrefixes");
1710-
if (encode_key) {
1711-
s->formatter->dump_string("Prefix", url_encode(pref_iter->first, false));
1712-
} else {
1713-
s->formatter->dump_string("Prefix", pref_iter->first);
1714-
}
1710+
dump_urlsafe(s, encode_key, "Prefix", pref_iter->first, false);
17151711

17161712
s->formatter->close_section();
17171713
}
@@ -1729,7 +1725,7 @@ void RGWListBucket_ObjStore_S3::send_versioned_response()
17291725
s->formatter->dump_string("KeyMarker", marker.name);
17301726
s->formatter->dump_string("VersionIdMarker", marker.instance);
17311727
if (is_truncated && !next_marker.empty()) {
1732-
s->formatter->dump_string("NextKeyMarker", next_marker.name);
1728+
dump_urlsafe(s ,encode_key, "NextKeyMarker", next_marker.name);
17331729
if (next_marker.instance.empty()) {
17341730
s->formatter->dump_string("NextVersionIdMarker", "null");
17351731
}
@@ -1752,14 +1748,7 @@ void RGWListBucket_ObjStore_S3::send_versioned_response()
17521748
s->formatter->dump_bool("IsDeleteMarker", iter->is_delete_marker());
17531749
}
17541750
rgw_obj_key key(iter->key);
1755-
if (encode_key) {
1756-
string key_name;
1757-
url_encode(key.name, key_name);
1758-
s->formatter->dump_string("Key", key_name);
1759-
}
1760-
else {
1761-
s->formatter->dump_string("Key", key.name);
1762-
}
1751+
dump_urlsafe(s ,encode_key, "Key", key.name);
17631752
string version_id = key.instance;
17641753
if (version_id.empty()) {
17651754
version_id = "null";
@@ -1807,11 +1796,7 @@ void RGWListBucket_ObjStore_S3::send_common_response()
18071796
s->formatter->dump_string("Prefix", prefix);
18081797
s->formatter->dump_int("MaxKeys", max);
18091798
if (!delimiter.empty()) {
1810-
if (encode_key) {
1811-
s->formatter->dump_string("Delimiter", url_encode(delimiter, false));
1812-
} else {
1813-
s->formatter->dump_string("Delimiter", delimiter);
1814-
}
1799+
dump_urlsafe(s, encode_key, "Delimiter", delimiter, false);
18151800
}
18161801
s->formatter->dump_string("IsTruncated", (max && is_truncated ? "true"
18171802
: "false"));
@@ -1821,11 +1806,7 @@ void RGWListBucket_ObjStore_S3::send_common_response()
18211806
for (pref_iter = common_prefixes.begin();
18221807
pref_iter != common_prefixes.end(); ++pref_iter) {
18231808
s->formatter->open_array_section("CommonPrefixes");
1824-
if (encode_key) {
1825-
s->formatter->dump_string("Prefix", url_encode(pref_iter->first, false));
1826-
} else {
1827-
s->formatter->dump_string("Prefix", pref_iter->first);
1828-
}
1809+
dump_urlsafe(s, encode_key, "Prefix", pref_iter->first, false);
18291810
s->formatter->close_section();
18301811
}
18311812
}
@@ -1866,13 +1847,6 @@ void RGWListBucket_ObjStore_S3::send_response()
18661847
for (iter = objs.begin(); iter != objs.end(); ++iter) {
18671848

18681849
rgw_obj_key key(iter->key);
1869-
std::string key_name;
1870-
1871-
if (encode_key) {
1872-
url_encode(key.name, key_name);
1873-
} else {
1874-
key_name = key.name;
1875-
}
18761850
/* conditionally format JSON in the obvious way--I'm unsure if
18771851
* AWS actually does this */
18781852
if (s->format == RGWFormat::XML) {
@@ -1881,7 +1855,7 @@ void RGWListBucket_ObjStore_S3::send_response()
18811855
// json
18821856
s->formatter->open_object_section("dummy");
18831857
}
1884-
s->formatter->dump_string("Key", key_name);
1858+
dump_urlsafe(s ,encode_key, "Key", key.name);
18851859
dump_time(s, "LastModified", iter->meta.mtime);
18861860
s->formatter->dump_format("ETag", "\"%s\"", iter->meta.etag.c_str());
18871861
s->formatter->dump_int("Size", iter->meta.accounted_size);
@@ -1905,7 +1879,7 @@ void RGWListBucket_ObjStore_S3::send_response()
19051879
}
19061880
s->formatter->dump_string("Marker", marker.name);
19071881
if (is_truncated && !next_marker.empty()) {
1908-
s->formatter->dump_string("NextMarker", next_marker.name);
1882+
dump_urlsafe(s, encode_key, "NextMarker", next_marker.name);
19091883
}
19101884
s->formatter->close_section();
19111885
rgw_flush_formatter_and_reset(s, s->formatter);
@@ -1941,14 +1915,7 @@ void RGWListBucket_ObjStore_S3v2::send_versioned_response()
19411915
s->formatter->dump_bool("IsDeleteContinuationToken", iter->is_delete_marker());
19421916
}
19431917
rgw_obj_key key(iter->key);
1944-
if (encode_key) {
1945-
string key_name;
1946-
url_encode(key.name, key_name);
1947-
s->formatter->dump_string("Key", key_name);
1948-
}
1949-
else {
1950-
s->formatter->dump_string("Key", key.name);
1951-
}
1918+
dump_urlsafe(s, encode_key, "Key", key.name);
19521919
string version_id = key.instance;
19531920
if (version_id.empty()) {
19541921
version_id = "null";
@@ -1986,11 +1953,7 @@ void RGWListBucket_ObjStore_S3v2::send_versioned_response()
19861953
for (pref_iter = common_prefixes.begin();
19871954
pref_iter != common_prefixes.end(); ++pref_iter) {
19881955
s->formatter->open_array_section("CommonPrefixes");
1989-
if (encode_key) {
1990-
s->formatter->dump_string("Prefix", url_encode(pref_iter->first, false));
1991-
} else {
1992-
s->formatter->dump_string("Prefix", pref_iter->first);
1993-
}
1956+
dump_urlsafe(s, encode_key, "Prefix", pref_iter->first, false);
19941957

19951958
s->formatter->dump_int("KeyCount",objs.size());
19961959
if (start_after_exist) {
@@ -2036,14 +1999,7 @@ void RGWListBucket_ObjStore_S3v2::send_response()
20361999
for (iter = objs.begin(); iter != objs.end(); ++iter) {
20372000
rgw_obj_key key(iter->key);
20382001
s->formatter->open_array_section("Contents");
2039-
if (encode_key) {
2040-
string key_name;
2041-
url_encode(key.name, key_name);
2042-
s->formatter->dump_string("Key", key_name);
2043-
}
2044-
else {
2045-
s->formatter->dump_string("Key", key.name);
2046-
}
2002+
dump_urlsafe(s, encode_key, "Key", key.name);
20472003
dump_time(s, "LastModified", iter->meta.mtime);
20482004
s->formatter->dump_format("ETag", "\"%s\"", iter->meta.etag.c_str());
20492005
s->formatter->dump_int("Size", iter->meta.accounted_size);
@@ -4147,11 +4103,7 @@ void RGWListBucketMultiparts_ObjStore_S3::send_response()
41474103
for (iter = uploads.begin(); iter != uploads.end(); ++iter) {
41484104
rgw::sal::MultipartUpload* upload = iter->get();
41494105
s->formatter->open_array_section("Upload");
4150-
if (encode_url) {
4151-
s->formatter->dump_string("Key", url_encode(upload->get_key(), false));
4152-
} else {
4153-
s->formatter->dump_string("Key", upload->get_key());
4154-
}
4106+
dump_urlsafe(s, encode_url, "Key", upload->get_key(), false);
41554107
s->formatter->dump_string("UploadId", upload->get_upload_id());
41564108
const ACLOwner& owner = upload->get_owner();
41574109
dump_owner(s, owner.get_id(), owner.get_display_name(), "Initiator");
@@ -4163,11 +4115,7 @@ void RGWListBucketMultiparts_ObjStore_S3::send_response()
41634115
if (!common_prefixes.empty()) {
41644116
s->formatter->open_array_section("CommonPrefixes");
41654117
for (const auto& kv : common_prefixes) {
4166-
if (encode_url) {
4167-
s->formatter->dump_string("Prefix", url_encode(kv.first, false));
4168-
} else {
4169-
s->formatter->dump_string("Prefix", kv.first);
4170-
}
4118+
dump_urlsafe(s, encode_url, "Prefix", kv.first, false);
41714119
}
41724120
s->formatter->close_section();
41734121
}

0 commit comments

Comments
 (0)