Skip to content

Commit abb57c9

Browse files
author
fangq
committed
massively speeding up loadjson for parsing large collection of unstructured small objects
git-svn-id: http://svn.code.sf.net/p/iso2mesh/code/trunk/jsonlab@487 786e58fb-9377-0410-9ff7-e4ac0ac0635c
1 parent afbedad commit abb57c9

File tree

3 files changed

+108
-131
lines changed

3 files changed

+108
-131
lines changed

‎loadjson.m‎

Lines changed: 5 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -109,41 +109,15 @@
109109
data=data{1};
110110
end
111111

112-
if(~isempty(data))
113-
if(isstruct(data)) % data can be a struct array
114-
data=jstruct2array(data);
115-
elseif(iscell(data))
116-
data=jcell2array(data);
117-
end
118-
end
119112
if(isfield(opt,'progressbar_'))
120113
close(opt.progressbar_);
121114
end
122115

123-
%%
124-
function newdata=jcell2array(data)
125-
len=length(data);
126-
newdata=data;
127-
for i=1:len
128-
if(isstruct(data{i}))
129-
newdata{i}=jstruct2array(data{i});
130-
elseif(iscell(data{i}))
131-
newdata{i}=jcell2array(data{i});
132-
end
133-
end
134-
135116
%%-------------------------------------------------------------------------
136117
function newdata=jstruct2array(data)
137118
fn=fieldnames(data);
138119
newdata=data;
139120
len=length(data);
140-
for i=1:length(fn) % depth-first
141-
for j=1:len
142-
if(isstruct(getfield(data(j),fn{i})))
143-
newdata(j)=setfield(newdata(j),fn{i},jstruct2array(getfield(data(j),fn{i})));
144-
end
145-
end
146-
end
147121
if(~isempty(strmatch('x0x5F_ArrayType_',fn)) && ~isempty(strmatch('x0x5F_ArrayData_',fn)))
148122
newdata=cell(len,1);
149123
for j=1:len
@@ -214,6 +188,9 @@
214188
end
215189
end
216190
parse_char('}');
191+
if(isstruct(object))
192+
object=jstruct2array(object);
193+
end
217194

218195
%%-------------------------------------------------------------------------
219196

@@ -405,9 +382,8 @@ function parse_char(c)
405382
%%-------------------------------------------------------------------------
406383

407384
function num = parse_number(varargin)
408-
global pos inStr len isoct
409-
currstr=inStr(pos:end);
410-
numstr=0;
385+
global pos inStr isoct
386+
currstr=inStr(pos:min(pos+30,end));
411387
if(isoct~=0)
412388
numstr=regexp(currstr,'^\s*-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+\-]?\d+)?','end');
413389
[num, one] = sscanf(currstr, '%f', 1);
@@ -439,13 +415,6 @@ function parse_char(c)
439415
return;
440416
case '{'
441417
val = parse_object(varargin{:});
442-
if isstruct(val)
443-
if(~isempty(strmatch('x0x5F_ArrayType_',fieldnames(val), 'exact')))
444-
val=jstruct2array(val);
445-
end
446-
elseif isempty(val)
447-
val = struct;
448-
end
449418
return;
450419
case {'-','0','1','2','3','4','5','6','7','8','9'}
451420
val = parse_number(varargin{:});
@@ -564,4 +533,3 @@ function error_pos(msg)
564533
if(endpos==0)
565534
error('unmatched "]"');
566535
end
567-

‎loadubjson.m‎

Lines changed: 7 additions & 94 deletions
Original file line numberDiff line numberDiff line change
@@ -94,15 +94,6 @@
9494
data=data{1};
9595
end
9696

97-
if(~isempty(data))
98-
if(isstruct(data)) % data can be a struct array
99-
data=jstruct2array(data);
100-
elseif(iscell(data))
101-
data=jcell2array(data);
102-
end
103-
end
104-
105-
10697
%%
10798
function newdata=parse_collection(id,data,obj)
10899

@@ -114,80 +105,6 @@
114105
end
115106
end
116107

117-
%%
118-
function newdata=jcell2array(data)
119-
len=length(data);
120-
newdata=data;
121-
for i=1:len
122-
if(isstruct(data{i}))
123-
newdata{i}=jstruct2array(data{i});
124-
elseif(iscell(data{i}))
125-
newdata{i}=jcell2array(data{i});
126-
end
127-
end
128-
129-
%%-------------------------------------------------------------------------
130-
function newdata=jstruct2array(data)
131-
fn=fieldnames(data);
132-
newdata=data;
133-
len=length(data);
134-
for i=1:length(fn) % depth-first
135-
for j=1:len
136-
if(isstruct(getfield(data(j),fn{i})))
137-
newdata(j)=setfield(newdata(j),fn{i},jstruct2array(getfield(data(j),fn{i})));
138-
end
139-
end
140-
end
141-
if(~isempty(strmatch('x0x5F_ArrayType_',fn)) && ~isempty(strmatch('x0x5F_ArrayData_',fn)))
142-
newdata=cell(len,1);
143-
for j=1:len
144-
ndata=cast(data(j).x0x5F_ArrayData_,data(j).x0x5F_ArrayType_);
145-
iscpx=0;
146-
if(~isempty(strmatch('x0x5F_ArrayIsComplex_',fn)))
147-
if(data(j).x0x5F_ArrayIsComplex_)
148-
iscpx=1;
149-
end
150-
end
151-
if(~isempty(strmatch('x0x5F_ArrayIsSparse_',fn)))
152-
if(data(j).x0x5F_ArrayIsSparse_)
153-
if(~isempty(strmatch('x0x5F_ArraySize_',fn)))
154-
dim=double(data(j).x0x5F_ArraySize_);
155-
if(iscpx && size(ndata,2)==4-any(dim==1))
156-
ndata(:,end-1)=complex(ndata(:,end-1),ndata(:,end));
157-
end
158-
if isempty(ndata)
159-
% All-zeros sparse
160-
ndata=sparse(dim(1),prod(dim(2:end)));
161-
elseif dim(1)==1
162-
% Sparse row vector
163-
ndata=sparse(1,ndata(:,1),ndata(:,2),dim(1),prod(dim(2:end)));
164-
elseif dim(2)==1
165-
% Sparse column vector
166-
ndata=sparse(ndata(:,1),1,ndata(:,2),dim(1),prod(dim(2:end)));
167-
else
168-
% Generic sparse array.
169-
ndata=sparse(ndata(:,1),ndata(:,2),ndata(:,3),dim(1),prod(dim(2:end)));
170-
end
171-
else
172-
if(iscpx && size(ndata,2)==4)
173-
ndata(:,3)=complex(ndata(:,3),ndata(:,4));
174-
end
175-
ndata=sparse(ndata(:,1),ndata(:,2),ndata(:,3));
176-
end
177-
end
178-
elseif(~isempty(strmatch('x0x5F_ArraySize_',fn)))
179-
if(iscpx && size(ndata,2)==2)
180-
ndata=complex(ndata(:,1),ndata(:,2));
181-
end
182-
ndata=reshape(ndata(:),data(j).x0x5F_ArraySize_);
183-
end
184-
newdata{j}=ndata;
185-
end
186-
if(len==1)
187-
newdata=newdata{1};
188-
end
189-
end
190-
191108
%%-------------------------------------------------------------------------
192109
function object = parse_object(varargin)
193110
parse_char('{');
@@ -226,6 +143,9 @@
226143
if(count==-1)
227144
parse_char('}');
228145
end
146+
if(isstruct(object))
147+
object=struct2jdata(object);
148+
end
229149

230150
%%-------------------------------------------------------------------------
231151
function [cid,len]=elem_info(type)
@@ -241,7 +161,7 @@
241161
%%-------------------------------------------------------------------------
242162

243163

244-
function [data adv]=parse_block(type,count,varargin)
164+
function [data, adv]=parse_block(type,count,varargin)
245165
global pos inStr isoct fileendian systemendian
246166
[cid,len]=elem_info(type);
247167
datastr=inStr(pos:pos+len*count-1);
@@ -282,7 +202,7 @@
282202
end
283203
if(~isempty(type))
284204
if(count>=0)
285-
[object adv]=parse_block(type,count,varargin{:});
205+
[object, adv]=parse_block(type,count,varargin{:});
286206
if(~isempty(dim))
287207
object=reshape(object,dim);
288208
end
@@ -292,7 +212,7 @@
292212
endpos=matching_bracket(inStr,pos);
293213
[cid,len]=elem_info(type);
294214
count=(endpos-pos)/len;
295-
[object adv]=parse_block(type,count,varargin{:});
215+
[object, adv]=parse_block(type,count,varargin{:});
296216
pos=pos+adv;
297217
parse_char(']');
298218
return;
@@ -426,13 +346,6 @@ function parse_char(c)
426346
return;
427347
case '{'
428348
val = parse_object(varargin{:});
429-
if isstruct(val)
430-
if(~isempty(strmatch('x0x5F_ArrayType_',fieldnames(val), 'exact')))
431-
val=jstruct2array(val);
432-
end
433-
elseif isempty(val)
434-
val = struct;
435-
end
436349
return;
437350
case {'i','U','I','l','L','d','D'}
438351
val = parse_number(varargin{:});
@@ -511,7 +424,7 @@ function error_pos(msg)
511424
end
512425
error('unmatched quotation mark');
513426
%%-------------------------------------------------------------------------
514-
function [endpos e1l e1r maxlevel] = matching_bracket(str,pos)
427+
function [endpos, e1l, e1r, maxlevel] = matching_bracket(str,pos)
515428
global arraytoken
516429
level=1;
517430
maxlevel=level;

‎struct2jdata.m‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
function newdata=struct2jdata(data,varargin)
2+
%
3+
% newdata=struct2jdata(data,opt,...)
4+
%
5+
% convert a JData object (in the form of a struct array) into an array
6+
%
7+
% authors:Qianqian Fang (fangq<at> nmr.mgh.harvard.edu)
8+
%
9+
% input:
10+
% data: a struct array. If data contains JData keywords in the first
11+
% level children, these fields are parsed and regrouped into a
12+
% data object (arrays, trees, graphs etc) based on JData
13+
% specification. The JData keywords are
14+
% "_ArrayType_", "_ArraySize_", "_ArrayData_"
15+
% "_ArrayIsSparse_", "_ArrayIsComplex_"
16+
% opt: (optional) a list of 'Param',value pairs for additional options
17+
% The supported options include
18+
% 'Recursive', if set to 1, will apply the conversion to
19+
% every child; 0 to disable
20+
%
21+
% output:
22+
% newdata: the covnerted data if the input data does contain a JData
23+
% structure; otherwise, the same as the input.
24+
%
25+
% examples:
26+
% obj=struct('_ArrayType_','double','_ArraySize_',[2 3],
27+
% '_ArrayIsSparse_',1 ,'_ArrayData_',null);
28+
% ubjdata=struct2jdata(obj);
29+
%
30+
% license:
31+
% BSD or GPL version 3, see LICENSE_{BSD,GPLv3}.txt files for details
32+
%
33+
% -- this function is part of JSONLab toolbox (http://iso2mesh.sf.net/cgi-bin/index.cgi?jsonlab)
34+
%
35+
36+
fn=fieldnames(data);
37+
newdata=data;
38+
len=length(data);
39+
if(jsonopt('Recursive',0,varargin{:})==1)
40+
for i=1:length(fn) % depth-first
41+
for j=1:len
42+
if(isstruct(getfield(data(j),fn{i})))
43+
newdata(j)=setfield(newdata(j),fn{i},jstruct2array(getfield(data(j),fn{i})));
44+
end
45+
end
46+
end
47+
end
48+
if(~isempty(strmatch('x0x5F_ArrayType_',fn)) && ~isempty(strmatch('x0x5F_ArrayData_',fn)))
49+
newdata=cell(len,1);
50+
for j=1:len
51+
ndata=cast(data(j).x0x5F_ArrayData_,data(j).x0x5F_ArrayType_);
52+
iscpx=0;
53+
if(~isempty(strmatch('x0x5F_ArrayIsComplex_',fn)))
54+
if(data(j).x0x5F_ArrayIsComplex_)
55+
iscpx=1;
56+
end
57+
end
58+
if(~isempty(strmatch('x0x5F_ArrayIsSparse_',fn)))
59+
if(data(j).x0x5F_ArrayIsSparse_)
60+
if(~isempty(strmatch('x0x5F_ArraySize_',fn)))
61+
dim=double(data(j).x0x5F_ArraySize_);
62+
if(iscpx && size(ndata,2)==4-any(dim==1))
63+
ndata(:,end-1)=complex(ndata(:,end-1),ndata(:,end));
64+
end
65+
if isempty(ndata)
66+
% All-zeros sparse
67+
ndata=sparse(dim(1),prod(dim(2:end)));
68+
elseif dim(1)==1
69+
% Sparse row vector
70+
ndata=sparse(1,ndata(:,1),ndata(:,2),dim(1),prod(dim(2:end)));
71+
elseif dim(2)==1
72+
% Sparse column vector
73+
ndata=sparse(ndata(:,1),1,ndata(:,2),dim(1),prod(dim(2:end)));
74+
else
75+
% Generic sparse array.
76+
ndata=sparse(ndata(:,1),ndata(:,2),ndata(:,3),dim(1),prod(dim(2:end)));
77+
end
78+
else
79+
if(iscpx && size(ndata,2)==4)
80+
ndata(:,3)=complex(ndata(:,3),ndata(:,4));
81+
end
82+
ndata=sparse(ndata(:,1),ndata(:,2),ndata(:,3));
83+
end
84+
end
85+
elseif(~isempty(strmatch('x0x5F_ArraySize_',fn)))
86+
if(iscpx && size(ndata,2)==2)
87+
ndata=complex(ndata(:,1),ndata(:,2));
88+
end
89+
ndata=reshape(ndata(:),data(j).x0x5F_ArraySize_);
90+
end
91+
newdata{j}=ndata;
92+
end
93+
if(len==1)
94+
newdata=newdata{1};
95+
end
96+
end

0 commit comments

Comments
 (0)