Skip to content

Commit d776bb4

Browse files
jimwwalkerdaverigby
jimwwalker
authored and
daverigby
committed
MB-16181: General improvements to collection stats
A number of tweaks to the documentation and the code. The primary improvements are: * Reduce superflous/duplicated data in keys, e.g. no need to include "manifest" repeatedly in collection-details. * Return maxTTl when defined via a number of 'views'. Change-Id: I3221e77057a7febbc651e8d490a26ab9397705b8 Reviewed-on: http://review.couchbase.org/c/kv_engine/+/133517 Tested-by: Build Bot <[email protected]> Reviewed-by: Dave Rigby <[email protected]>
1 parent de9b0c3 commit d776bb4

File tree

6 files changed

+81
-69
lines changed

6 files changed

+81
-69
lines changed

‎engines/ep/docs/stats.org‎

Lines changed: 19 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1304,6 +1304,7 @@ collection-id.
13041304

13051305
| sid:cid:disk_size | Approximate disk-usage of the collection. Note the sum of all collection disk-sizes does not equal the bucket disk usage |
13061306
| sid:cid:items | Number of items stored in the collection. |
1307+
| sid:cid:maxTTL | The Time-To-Live value for the collection, omitted if none defined. |
13071308
| sid:cid:mem_used | Approximate memory-usage of the collection. Note the sum of all collection mem_used does not equal the bucket mem_used. |
13081309
| sid:cid:name | The collection's name. |
13091310
| sid:cid:ops_delete | The number of delete operations performed against the collection. |
@@ -1321,14 +1322,14 @@ disk-size is initialised to the total disk used by the bucket.
13211322
prefixed with the vbucket ID as "vb_x", where x is a decimal value and may also
13221323
include the collection-id encoded as a 0x hexadecimal value (cid used in table).
13231324

1324-
| vb_x:collection:cid:entry:high_seqno | The high-seqno of the collection. |
1325-
| vb_x:collection:cid:entry:items | The number of items the collection stores in this vbucket. |
1326-
| vb_x:collection:cid:entry:persisted_high_seqno | The highest persisted seqno. |
1327-
| vb_x:collection:cid:entry:scope | The collections scope (as an 0x id). |
1328-
| vb_x:collection:cid:entry:start_seqno | The start seqno of the collection, the seqno when it was created. |
1329-
| vb_x:collection:cid:entry:maxTTL | The Time-To-Live value for the collection, omitted if none defined.|
1330-
| vb_x:manifest:entries | The number of collections the vbucket knows about. |
1331-
| vb_x:manifest:uid | The manifest last used to update the vbucket. |
1325+
| vb_x:cid:high_seqno | The high-seqno of the collection. |
1326+
| vb_x:cid:items | The number of items the collection stores in this vbucket. |
1327+
| vb_x:cid:persisted_high_seqno | The highest persisted seqno. |
1328+
| vb_x:cid:scope | The collection's scope (as an 0x id). |
1329+
| vb_x:cid:maxTTL | The Time-To-Live value for the collection, omitted if none defined.|
1330+
| vb_x:cid:start_seqno | The start seqno of the collection, the seqno when it was created. |
1331+
| vb_x:collections | The number of collections the vbucket knows about. |
1332+
| vb_x:manifest_uid | The id of the manifest last used to update the vbucket. |
13321333

13331334
*** Scopes
13341335

@@ -1338,6 +1339,12 @@ hexadecimal value, for collections within a scope they are prefixed with
13381339
scope-id and collection-id as a 0x prefixed hexadecimal value. For brevity,
13391340
'sid' and 'cid' are used for scope-id and collection-id.
13401341

1342+
When a specific scope is selected, each collection within the scope is returned.
1343+
When the scope (no argument) key is used only the names of the collections in
1344+
each scope are returned. The sid:cid stats returned within the scopes view are
1345+
the same values (and definitions) as the keys/value returned from "collections"
1346+
and "collections-byid".
1347+
13411348
| sid:cid:name | The name of a collection in the scope, multiple names maybe returned. |
13421349
| sid:collections | The count of collections in the scope. |
13431350
| sid:disk_size | The sum of all collection 'disk_size'. |
@@ -1354,12 +1361,10 @@ allows a single vbucket to be inspected. Keys returned are prefixed with the
13541361
vbucket ID as "vb_x", where x is a decimal value and may also include the
13551362
scope/collection-id encoded as a 0x hexadecimal value (sid/cid used in table).
13561363

1357-
Some stat keys also include the iteration index from the internal data structure, shown as n.
1358-
1359-
| vb_x:manifest:scope:sid:collection:n | The id of a collection in scope sid. |
1360-
| vb_x:manifest:scopes | The number of scopes. |
1361-
| vb_x:manifest:scopes:n | The id of the scope. |
1362-
| vb_x:manifest:uid | The manifest last used to update the vbucket. |
1364+
| vb_x:scopes | The number of scopes. |
1365+
| vb_x:sid | All of the known scope-ids returned, the value is the index position from the internal container |
1366+
| vb_x:sid:cid:items | The item count of a collection, repeated for all collections. |
1367+
| vb_x:manifest_uid | The id of the manifest last used to update the vbucket. |
13631368

13641369
** Stats Reset
13651370

‎engines/ep/src/collections/manager.cc‎

Lines changed: 31 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -670,7 +670,7 @@ cb::EngineErrorGetScopeIDResult Collections::Manager::doOneScopeStats(
670670
Expects(itr != current->end());
671671
const auto& [cid, collection] = *itr;
672672
cachedStats.addStatsForCollection(
673-
{}, cid, collection, add_stat, cookie);
673+
scope, cid, collection, add_stat, cookie);
674674
}
675675
return res;
676676
}
@@ -704,8 +704,8 @@ Collections::CachedStats::CachedStats(
704704
accumulatedStats(std::move(accumulatedStats)) {
705705
}
706706
void Collections::CachedStats::addStatsForCollection(
707-
const std::optional<Scope>& scope,
708-
const CollectionID& cid,
707+
const Scope& scope,
708+
CollectionID cid,
709709
const Manifest::Collection& collection,
710710
const AddStatFn& add_stat,
711711
const void* cookie) {
@@ -721,16 +721,31 @@ void Collections::CachedStats::addStatsForCollection(
721721
format_to(buf, "{}:{}:name", scopeID.to_string(), cid.to_string());
722722
add_stat({buf.data(), buf.size()}, collection.name, cookie);
723723

724-
// add scope name stat?
725-
if (scope) {
726-
buf.resize(0);
727-
format_to(
728-
buf, "{}:{}:scope_name", scopeID.to_string(), cid.to_string());
729-
add_stat({buf.data(), buf.size()}, scope.value().name, cookie);
724+
// add scope name stat
725+
buf.resize(0);
726+
format_to(buf, "{}:{}:scope_name", scopeID.to_string(), cid.to_string());
727+
add_stat({buf.data(), buf.size()}, scope.name, cookie);
728+
// ttl (requires a search in the Scope vector)
729+
for (const auto& collection : scope.collections) {
730+
if (collection.id == cid) {
731+
if (collection.maxTtl) {
732+
buf.resize(0);
733+
format_to(buf,
734+
"{}:{}:maxTTL",
735+
scopeID.to_string(),
736+
cid.to_string());
737+
fmt::memory_buffer value;
738+
format_to(value, "{}", collection.maxTtl.value().count());
739+
add_stat({buf.data(), buf.size()},
740+
{value.data(), value.size()},
741+
cookie);
742+
}
743+
break;
744+
}
730745
}
731746
}
732747

733-
void Collections::CachedStats::addStatsForScope(const ScopeID& sid,
748+
void Collections::CachedStats::addStatsForScope(ScopeID sid,
734749
const Scope& scope,
735750
const AddStatFn& add_stat,
736751
const void* cookie) {
@@ -748,6 +763,12 @@ void Collections::CachedStats::addStatsForScope(const ScopeID& sid,
748763
fmt::memory_buffer buf;
749764
format_to(buf, "{}:name", sid.to_string());
750765
add_stat({buf.data(), buf.size()}, scope.name, cookie);
766+
// add scope collection count
767+
buf.resize(0);
768+
format_to(buf, "{}:collections", sid.to_string());
769+
fmt::memory_buffer value;
770+
format_to(value, "{}", scope.collections.size());
771+
add_stat({buf.data(), buf.size()}, {value.data(), value.size()}, cookie);
751772
}
752773

753774
void Collections::CachedStats::addAggregatedCollectionStats(

‎engines/ep/src/collections/manager.h‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -54,8 +54,8 @@ class CachedStats {
5454
* @param add_stat memcached callback which will be called with each stat
5555
* @param cookie
5656
*/
57-
void addStatsForCollection(const std::optional<Scope>& scope,
58-
const CollectionID& cid,
57+
void addStatsForCollection(const Scope& scope,
58+
CollectionID cid,
5959
const Manifest::Collection& collection,
6060
const AddStatFn& add_stat,
6161
const void* cookie);
@@ -68,7 +68,7 @@ class CachedStats {
6868
* @param add_stat memcached callback which will be called with each stat
6969
* @param cookie
7070
*/
71-
void addStatsForScope(const ScopeID& sid,
71+
void addStatsForScope(ScopeID sid,
7272
const Scope& scope,
7373
const AddStatFn& add_stat,
7474
const void* cookie);

‎engines/ep/src/collections/vbucket_manifest.cc‎

Lines changed: 16 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -812,7 +812,7 @@ bool Manifest::addCollectionStats(Vbid vbid,
812812
try {
813813
const int bsize = 512;
814814
char buffer[bsize];
815-
checked_snprintf(buffer, bsize, "vb_%d:manifest:entries", vbid.get());
815+
checked_snprintf(buffer, bsize, "vb_%d:collections", vbid.get());
816816
add_casted_stat(buffer, map.size(), add_stat, cookie);
817817
checked_snprintf(buffer, bsize, "vb_%d:manifest:uid", vbid.get());
818818
add_casted_stat(buffer, manifestUid, add_stat, cookie);
@@ -839,9 +839,9 @@ bool Manifest::addScopeStats(Vbid vbid,
839839
const int bsize = 512;
840840
char buffer[bsize];
841841
try {
842-
checked_snprintf(buffer, bsize, "vb_%d:manifest:scopes", vbid.get());
842+
checked_snprintf(buffer, bsize, "vb_%d:scopes", vbid.get());
843843
add_casted_stat(buffer, scopes.size(), add_stat, cookie);
844-
checked_snprintf(buffer, bsize, "vb_%d:manifest:uid", vbid.get());
844+
checked_snprintf(buffer, bsize, "vb_%d:manifest_uid", vbid.get());
845845
add_casted_stat(buffer, manifestUid, add_stat, cookie);
846846
} catch (const std::exception& e) {
847847
EP_LOG_WARN(
@@ -852,24 +852,26 @@ bool Manifest::addScopeStats(Vbid vbid,
852852
return false;
853853
}
854854

855-
// We'll also print the iteration index of each scope and collection.
855+
// Dump the scopes container, which only stores scope-identifiers, so that
856+
// we have a key and value we include the iteration index as the value.
856857
int i = 0;
857-
for (auto it = scopes.begin(); it != scopes.end(); it++, i++) {
858+
for (auto sid : scopes) {
858859
checked_snprintf(
859-
buffer, bsize, "vb_%d:manifest:scopes:%d", vbid.get(), i);
860-
add_casted_stat(buffer, it->to_string().c_str(), add_stat, cookie);
860+
buffer, bsize, "vb_%d:%s", vbid.get(), sid.to_string().c_str());
861+
add_casted_stat(buffer, i++, add_stat, cookie);
861862
}
862863

863-
i = 0;
864-
for (auto it = map.begin(); it != map.end(); it++, i++) {
864+
// Dump all collections and how they map to a scope. Stats requires unique
865+
// keys, so cannot use the cid as the value (as key won't be unique), so
866+
// return anything from the entry for the value, we choose item count.
867+
for (const auto& [cid, entry] : map) {
865868
checked_snprintf(buffer,
866869
bsize,
867-
"vb_%d:manifest:scope:%s:collection:%d",
870+
"vb_%d:%s:%s:items",
868871
vbid.get(),
869-
it->second.getScopeID().to_string().c_str(),
870-
i);
871-
add_casted_stat(
872-
buffer, it->first.to_string().c_str(), add_stat, cookie);
872+
entry.getScopeID().to_string().c_str(),
873+
cid.to_string().c_str());
874+
add_casted_stat(buffer, entry.getDiskCount(), add_stat, cookie);
873875
}
874876

875877
return true;

‎engines/ep/src/collections/vbucket_manifest_entry.cc‎

Lines changed: 11 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -67,43 +67,28 @@ bool Collections::VB::ManifestEntry::addStats(const std::string& cid,
6767
try {
6868
const int bsize = 512;
6969
char buffer[bsize];
70-
checked_snprintf(buffer,
71-
bsize,
72-
"vb_%d:collection:%s:entry:scope",
73-
vbid.get(),
74-
cid.c_str());
70+
checked_snprintf(
71+
buffer, bsize, "vb_%d:%s:scope", vbid.get(), cid.c_str());
7572
add_casted_stat(buffer, getScopeID().to_string(), add_stat, cookie);
76-
checked_snprintf(buffer,
77-
bsize,
78-
"vb_%d:collection:%s:entry:start_seqno",
79-
vbid.get(),
80-
cid.c_str());
73+
checked_snprintf(
74+
buffer, bsize, "vb_%d:%s:start_seqno", vbid.get(), cid.c_str());
8175
add_casted_stat(buffer, getStartSeqno(), add_stat, cookie);
82-
checked_snprintf(buffer,
83-
bsize,
84-
"vb_%d:collection:%s:entry:high_seqno",
85-
vbid.get(),
86-
cid.c_str());
76+
checked_snprintf(
77+
buffer, bsize, "vb_%d:%s:high_seqno", vbid.get(), cid.c_str());
8778
add_casted_stat(buffer, getHighSeqno(), add_stat, cookie);
8879
checked_snprintf(buffer,
8980
bsize,
90-
"vb_%d:collection:%s:entry:persisted_high_seqno",
81+
"vb_%d:%s:persisted_high_seqno",
9182
vbid.get(),
9283
cid.c_str());
9384
add_casted_stat(buffer, getPersistedHighSeqno(), add_stat, cookie);
94-
checked_snprintf(buffer,
95-
bsize,
96-
"vb_%d:collection:%s:entry:items",
97-
vbid.get(),
98-
cid.c_str());
85+
checked_snprintf(
86+
buffer, bsize, "vb_%d:%s:items", vbid.get(), cid.c_str());
9987
add_casted_stat(buffer, getDiskCount(), add_stat, cookie);
10088

10189
if (getMaxTtl()) {
102-
checked_snprintf(buffer,
103-
bsize,
104-
"vb_%d:collection:%s:entry:maxTTL",
105-
vbid.get(),
106-
cid.c_str());
90+
checked_snprintf(
91+
buffer, bsize, "vb_%d:%s:maxTTL", vbid.get(), cid.c_str());
10792
add_casted_stat(buffer, getMaxTtl().value().count(), add_stat, cookie);
10893
}
10994
return true;

‎engines/ep/tests/ep_test_apis.cc‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -828,8 +828,7 @@ void verify_all_vb_seqnos(EngineIface* h,
828828
if (cid) {
829829
// Get high seqno for the collection in the vBucket
830830
std::string vb_stat_seqno("vb_" + std::to_string(vb_start + i) +
831-
":collection:" + cid->to_string() +
832-
":entry:high_seqno");
831+
":" + cid->to_string() + ":high_seqno");
833832
high_seqno_vb = get_ull_stat(
834833
h, vb_stat_seqno.c_str(), "collections-details");
835834
} else {

0 commit comments

Comments
 (0)