Skip to content

Commit 51e3a66

Browse files
linevlinev
committed
Intermediate commit
Try to correctly read TBasket::fEntryOffset. Seems to be, there is no good solution for the reading of empty STL vectors.
1 parent ab1fafe commit 51e3a66

File tree

2 files changed

+117
-38
lines changed

2 files changed

+117
-38
lines changed

‎scripts/JSRootIOEvolution.js‎

Lines changed: 31 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,8 @@
4848
// constants of bits in version
4949
kStreamedMemberWise : JSROOT.BIT(14),
5050

51+
kSplitCollectionOfPointers: 100,
52+
5153
// map of user-streamer function like func(buf,obj)
5254
// or alias (classname) which can be used to read that function
5355
// or list of read functions
@@ -598,6 +600,8 @@
598600
if (obj.fLast > obj.fBufferSize) obj.fBufferSize = obj.fLast;
599601
var flag = this.ntoi1();
600602

603+
console.log('READ BASKET', ver, flag, obj.fNevBuf, obj.fBufferSize, obj.fNevBufSize, obj.fLast);
604+
601605
if (flag===0) return obj;
602606

603607
if ((flag % 10) != 2) {
@@ -608,8 +612,10 @@
608612
obj.fEntryOffset[i] &= ~kDisplacementMask;
609613
}
610614

611-
if (flag>40)
615+
if (flag>40) {
612616
obj.fDisplacement = this.ReadFastArray(this.ntoi4(), JSROOT.IO.kInt);
617+
console.log('READ DISPLACEMENT', obj.fDisplacement.length);
618+
}
613619
}
614620

615621
if ((flag === 1) || (flag > 10)) {
@@ -1979,7 +1985,7 @@
19791985
if ((element._typename === 'TStreamerSTLstring') ||
19801986
(member.typename == "string") || (member.typename == "string*")) {
19811987
member.readelem = function(buf) { return buf.ReadTString(); };
1982-
member.read_empty_stl_version = false;
1988+
// member.read_empty_stl_version = false;
19831989
} else
19841990
if ((stl === JSROOT.IO.kSTLvector) || (stl === JSROOT.IO.kSTLlist) ||
19851991
(stl === JSROOT.IO.kSTLdeque) || (stl === JSROOT.IO.kSTLset) ||
@@ -2050,7 +2056,7 @@
20502056

20512057
member.streamer = JSROOT.IO.GetPairStreamer(member.si, member.pairtype, file);
20522058

2053-
member.read_empty_stl_version = true; // in branch reading read version even for empty container, vector does not have it
2059+
// member.read_empty_stl_version = true; // in branch reading read version even for empty container, vector does not have it
20542060

20552061
if (!member.streamer || (member.streamer.length!==2)) {
20562062
JSROOT.console('Fail to build streamer for pair ' + member.pairtype);
@@ -2078,19 +2084,37 @@
20782084
member.read_version = function(buf, cnt) {
20792085
// read version, check member-wise flag; if any, read version for contained object
20802086

2087+
console.log('start reading version', buf.o, buf.remain(), 'cnt', cnt, 'read_empty', this.read_empty_stl_version);
2088+
20812089
// workaround - in some cases version is not written for empty container
2082-
if ((cnt===0) && ((buf.remain()<6) || this.read_empty_stl_version === false)) return null;
2090+
if ((cnt===0) && ((buf.remain()<6) || this.read_empty_stl_version === false)) {
2091+
if (this.read_empty_stl_version) throw new Error("SKIP0 VERSION WHEN SHOULD READ " + this.typename + " - " + this.name);
2092+
return null;
2093+
}
20832094

20842095
var o = buf.o, ver = buf.ReadVersion();
20852096

2086-
if ((cnt===0) && (ver.bytecnt===0)) return ver;
2097+
console.log('cnt',cnt, 'ver', ver, 'read_empty', this.read_empty_stl_version);
20872098

20882099
this.member_wise = ((ver.val & JSROOT.IO.kStreamedMemberWise) !== 0);
20892100

2101+
if ((cnt===0) && (ver.bytecnt===0) && !this.member_wise) {
2102+
2103+
if (this.read_empty_stl_version!==true) throw new Error("READ ZERO WHEN SHOULD NOT " + this.typename + " - " + this.name + ' ver ' + ver.val);
2104+
2105+
return ver;
2106+
}
2107+
20902108
// workaround - in some cases version is not written for empty container
20912109
if (cnt===0)
20922110
if (((ver.bytecnt!==6) && (ver.bytecnt!=2) && !this.read_empty_stl_version) ||
2093-
(ver.bytecnt && (ver.bytecnt > (this.member_wise ? 12 : 6)))) { buf.o = o; return null; }
2111+
(ver.bytecnt && (ver.bytecnt > (this.member_wise ? 12 : 6)))) {
2112+
if (this.read_empty_stl_version) throw new Error("SKIP1 VERSION WHEN SHOULD READ " + this.typename + " - " + this.name + ' bytecnt ' + ver.bytecnt);
2113+
buf.o = o;
2114+
return null;
2115+
}
2116+
2117+
if ((cnt===0) && this.read_empty_stl_version===false) throw new Error("READ VERSION WHEN SHOULD SKIP " + this.typename + " - " + this.name);
20942118

20952119
this.stl_version = undefined;
20962120
if (this.member_wise) {
@@ -2730,7 +2754,7 @@
27302754
if (n===0) return []; // for empty vector no need to search splitted streamers
27312755

27322756
if (n>200000) {
2733-
throw new Error('member-wise streaming for of ' + this.conttype + " num " + n);
2757+
throw new Error('member-wise streaming of ' + this.conttype + " num " + n + ' member ' + this.name);
27342758
return [];
27352759
}
27362760

‎scripts/JSRootTree.js‎

Lines changed: 86 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1386,6 +1386,7 @@
13861386
type: 0, // keep type identifier
13871387
curr_entry: -1, // last processed entry
13881388
raw : null, // raw buffer for reading
1389+
basket : null, // current basket object
13891390
curr_basket: 0, // number of basket used for processing
13901391
read_entry: -1, // last entry which is already read
13911392
staged_entry: -1, // entry which is staged for reading
@@ -1399,13 +1400,24 @@
13991400
staged_prev: 0, // entry limit of previous I/O request
14001401
staged_now: 0, // entry limit of current I/O request
14011402
progress_showtm: 0, // last time when progress was showed
1403+
have_to_read_empty_stl_branch: false, // by default splitted container branch read version if empty container
14021404
GetBasketEntry : function(k) {
14031405
if (!this.branch || (k > this.branch.fMaxBaskets)) return 0;
14041406
var res = (k < this.branch.fMaxBaskets) ? this.branch.fBasketEntry[k] : 0;
14051407
if (res) return res;
14061408
var bskt = (k>0) ? this.branch.fBaskets.arr[k-1] : null;
14071409
return bskt ? (this.branch.fBasketEntry[k-1] + bskt.fNevBuf) : 0;
14081410
},
1411+
LocateBuffer : function(entry) {
1412+
// locate buffer at proper position
1413+
var shift = entry - this.first_entry, off;
1414+
if (this.basket.fEntryOffset) {
1415+
off = this.basket.fEntryOffset[shift];
1416+
} else {
1417+
off = this.basket.fNevBufSize * shift;
1418+
}
1419+
this.raw.locate((this.raw.entry_shift || 0) + off);
1420+
},
14091421
GetTarget : function(tgtobj) {
14101422
if (!this.tgt) return tgtobj;
14111423
for (var k=0;k<this.tgt.length;++k) {
@@ -1415,7 +1427,6 @@
14151427
}
14161428
return tgtobj;
14171429
}
1418-
14191430
};
14201431

14211432
// last basket can be stored directly with the branch
@@ -1630,8 +1641,13 @@
16301641
} else
16311642
if (is_brelem && (nb_leaves <= 1)) {
16321643

1644+
item.have_to_read_empty_stl_branch = (branch.fType === JSROOT.BranchType.kClonesMemberNode);
1645+
16331646
elem = JSROOT.IO.FindBrachStreamerElement(branch, handle.file);
16341647

1648+
if (elem)
1649+
console.log('ELEMENT', elem.fName, elem.fType, elem.fTypeName, 'READ_EMPTY', item.have_to_read_empty_stl_branch);
1650+
16351651
// this is basic type - can try to solve problem differently
16361652
if (!elem && branch.fStreamerType && (branch.fStreamerType < 20)) {
16371653
elem = JSROOT.IO.CreateStreamerElement(target_name, branch.fStreamerType);
@@ -1747,6 +1763,8 @@
17471763
// STL branch provides special function for the reading
17481764
member.func = member.objs_branch_func;
17491765

1766+
member.read_empty_stl_version = item.have_to_read_empty_stl_branch;
1767+
17501768
} else {
17511769
member.func0 = member.func;
17521770

@@ -1822,13 +1840,16 @@
18221840
// function provided by normal I/O
18231841
member.func = member.branch_func;
18241842
member.stl_size = item_cnt.name;
1843+
member.read_empty_stl_version = item.have_to_read_empty_stl_branch;
1844+
18251845
} else
18261846
if ((elem.fType === JSROOT.IO.kStreamLoop) || (elem.fType === JSROOT.IO.kOffsetL+JSROOT.IO.kStreamLoop)) {
18271847
if (item_cnt2) {
18281848
// special solution for kStreamLoop
18291849
member.stl_size = item_cnt.name;
18301850
member.cntname = item_cnt2.name;
18311851
member.func = member.branch_func; // this is special function, provided by base I/O
1852+
member.read_empty_stl_version = item.have_to_read_empty_stl_branch;
18321853
} else {
18331854
member.cntname = item_cnt.name;
18341855
}
@@ -1939,7 +1960,7 @@
19391960
if (!isNaN(args.firstentry) && (args.firstentry>handle.firstentry) && (args.firstentry < handle.lastentry))
19401961
handle.process_min = args.firstentry;
19411962

1942-
handle.staged_now = handle.process_min;
1963+
handle.current_entry = handle.staged_now = handle.process_min;
19431964

19441965
if (!isNaN(args.numentries) && (args.numentries>0)) {
19451966
var max = handle.process_min + args.numentries;
@@ -2021,6 +2042,23 @@
20212042
handle.selector.ShowProgress(portion);
20222043
}
20232044

2045+
function ReadBasketEntryOffset(branch, basket, buf) {
2046+
if (branch.fEntryOffsetLen <= 0) return;
2047+
2048+
// ready entry offest len when necessary
2049+
2050+
buf.locate(basket.fLast - buf.raw_shift);
2051+
2052+
basket.fEntryOffset = buf.ReadFastArray(buf.ntoi4(), JSROOT.IO.kInt);
2053+
if (!basket.fEntryOffset) basket.fEntryOffset = [ basket.fKeylen ];
2054+
2055+
if (buf.remain() > 0)
2056+
basket.fDisplacement = buf.ReadFastArray(buf.ntoi4(), JSROOT.IO.kInt);
2057+
2058+
// rollback buffer - not needed in the future
2059+
buf.locate(buf.raw_shift);
2060+
}
2061+
20242062
function ProcessBlobs(blobs) {
20252063
if (!blobs || ((places.length>2) && (blobs.length*2 !== places.length)))
20262064
return JSROOT.CallBack(baskets_call_back, null);
@@ -2042,19 +2080,35 @@
20422080

20432081
// items[k].obj = basket; // keep basket object itself if necessary
20442082

2045-
bitems[k].fNevBuf = basket.fNevBuf; // only number of entries in the basket are relevant for the moment
2083+
bitems[k].bskt_obj = basket; // only number of entries in the basket are relevant for the moment
2084+
2085+
console.log('CHECK', basket.fKeylen, basket.fObjlen, basket.fNbytes);
20462086

20472087
if (basket.fKeylen + basket.fObjlen === basket.fNbytes) {
20482088
// use data from original blob
2049-
bitems[k].raw = buf;
2089+
buf.raw_shift = 0;
2090+
buf.entry_shift = basket.fKeylen;
2091+
20502092
} else {
20512093
// unpack data and create new blob
20522094
var objblob = JSROOT.R__unzip(blob, basket.fObjlen, false, buf.o);
20532095

2054-
if (objblob) bitems[k].raw = JSROOT.CreateTBuffer(objblob, 0, handle.file);
2055-
2056-
if (bitems[k].raw) bitems[k].raw.fTagOffset = basket.fKeylen;
2096+
if (objblob) {
2097+
buf = JSROOT.CreateTBuffer(objblob, 0, handle.file);
2098+
buf.raw_shift = basket.fKeylen;
2099+
buf.fTagOffset = basket.fKeylen;
2100+
} else {
2101+
throw new Error('FAIL TO UNPACK');
2102+
}
20572103
}
2104+
2105+
bitems[k].raw = buf; // here already unpacket buffer
2106+
2107+
console.log('BUFFER', buf.remain());
2108+
2109+
ReadBasketEntryOffset(bitems[k].branch, basket, buf);
2110+
2111+
console.log('Extract RAW ', buf.remain(), 'last', basket.fLast - basket.fKeylen, 'objlen', basket.fObjlen);
20582112
}
20592113

20602114
if (ExtractPlaces())
@@ -2093,10 +2147,10 @@
20932147
var lmt = elem.GetBasketEntry(k+1),
20942148
not_needed = (lmt <= handle.process_min);
20952149

2096-
for (var d=0;d<elem.ascounter.length;++d) {
2097-
var dep = handle.arr[elem.ascounter[d]]; // dependent element
2098-
if (dep.first_readentry < lmt) not_needed = false; // check that counter provide required data
2099-
}
2150+
//for (var d=0;d<elem.ascounter.length;++d) {
2151+
// var dep = handle.arr[elem.ascounter[d]]; // dependent element
2152+
// if (dep.first_readentry < lmt) not_needed = false; // check that counter provide required data
2153+
//}
21002154

21012155
if (not_needed) continue; // if that basket not required, check next
21022156

@@ -2121,7 +2175,12 @@
21212175
bitem.raw.locate(0); // reset pointer - same branch may be read several times
21222176
else
21232177
bitem.raw = JSROOT.CreateTBuffer(null, 0, handle.file); // create dummy buffer - basket has no data
2124-
bitem.fNevBuf = bskt.fNevBuf;
2178+
bitem.raw.raw_shift = 0;
2179+
2180+
if (bskt.fBufferRef)
2181+
ReadBasketEntryOffset(elem.branch, bskt, bitem.raw);
2182+
2183+
bitem.bskt_obj = bskt;
21252184
is_direct = true;
21262185
elem.baskets[k] = bitem;
21272186
} else {
@@ -2183,7 +2242,10 @@
21832242

21842243
elem = handle.arr[n];
21852244

2186-
if (!elem.raw) {
2245+
if (!elem.raw || !elem.basket || (elem.first_entry + elem.basket.fNevBuf <= handle.current_entry)) {
2246+
delete elem.raw;
2247+
delete elem.basket;
2248+
21872249
if ((elem.curr_basket >= elem.numbaskets)) {
21882250
if (n==0) return handle.selector.Terminate(true);
21892251
continue; // ignore non-master branch
@@ -2205,21 +2267,20 @@
22052267
}
22062268

22072269
elem.raw = bitem.raw;
2208-
elem.nev = bitem.fNevBuf; // number of entries in raw buffer
2209-
elem.current_entry = elem.GetBasketEntry(bitem.basket);
2270+
elem.basket = bitem.bskt_obj;
2271+
// elem.nev = bitem.fNevBuf; // number of entries in raw buffer
2272+
elem.first_entry = elem.GetBasketEntry(bitem.basket);
22102273

22112274
bitem.raw = null; // remove reference on raw buffer
22122275
bitem.branch = null; // remove reference on the branch
2276+
bitem.bskt_obj = null; // remove reference on the branch
22132277
elem.baskets[elem.curr_basket++] = undefined; // remove from array
22142278
}
22152279

2216-
min_curr = Math.min(min_curr, elem.current_entry);
2217-
loopentries = Math.min(loopentries, elem.nev); // define how much entries can be processed before next raw buffer will be finished
2280+
// define how much entries can be processed before next raw buffer will be finished
2281+
loopentries = Math.min(loopentries, elem.first_entry + elem.basket.fNevBuf - handle.current_entry);
22182282
}
22192283

2220-
// assign first entry which can be analyzed
2221-
if (handle.current_entry < 0) handle.current_entry = min_curr;
2222-
22232284
// second loop extracts all required data
22242285

22252286
// do not read too much
@@ -2233,7 +2294,6 @@
22332294
elem = handle.arr[n];
22342295
elem.arrmember.arrlength = loopentries;
22352296
elem.arrmember.func(elem.raw, handle.selector.tgtarr);
2236-
elem.current_entry += loopentries;
22372297

22382298
elem.raw = null;
22392299
}
@@ -2250,19 +2310,14 @@
22502310
for (n=0;n<handle.arr.length;++n) {
22512311
elem = handle.arr[n];
22522312

2253-
if (handle.current_entry === elem.current_entry) {
2254-
// read only element where entry id matches
2255-
2256-
elem.member.func(elem.raw, elem.GetTarget(handle.selector.tgtobj));
2313+
// locate buffer offest at proper place
2314+
elem.LocateBuffer(handle.current_entry);
22572315

2258-
elem.current_entry++;
2259-
2260-
if (--elem.nev <= 0) elem.raw = null;
2261-
}
2316+
// read only element where entry id matches
2317+
elem.member.func(elem.raw, elem.GetTarget(handle.selector.tgtobj));
22622318
}
22632319

2264-
if (handle.current_entry >= handle.process_min)
2265-
handle.selector.Process(handle.current_entry);
2320+
handle.selector.Process(handle.current_entry);
22662321

22672322
handle.current_entry++;
22682323

0 commit comments

Comments
 (0)