@@ -6,8 +6,24 @@ var filterableMixin = require('ampersand-collection-filterable');
6
6
var SampledDocumentCollection = require ( './sampled-document-collection' ) ;
7
7
var es = require ( 'event-stream' ) ;
8
8
var debug = require ( 'debug' ) ( 'scout:models:schema' ) ;
9
+ var debugMetrics = require ( 'debug' ) ( 'scout:metrics' ) ;
9
10
var app = require ( 'ampersand-app' ) ;
10
11
12
+ // @todo : stub for metrics module. currently just logs debug messages with scout:metrics marker
13
+ var metrics = {
14
+ track : function ( label , err , obj ) {
15
+ if ( ! obj ) {
16
+ obj = err ;
17
+ err = null ;
18
+ }
19
+ if ( err ) {
20
+ debugMetrics ( 'metrics error: ' , err , obj ) ;
21
+ } else {
22
+ debugMetrics ( 'metrics log: ' , obj ) ;
23
+ }
24
+ }
25
+ } ;
26
+
11
27
/**
12
28
* wrapping mongodb-schema's FieldCollection with a filterable mixin
13
29
*/
@@ -25,6 +41,8 @@ module.exports = Schema.extend({
25
41
}
26
42
} ,
27
43
session : {
44
+ // total number of documents counted under the given query
45
+ total : 'number' ,
28
46
is_fetching : {
29
47
type : 'boolean' ,
30
48
default : false
@@ -96,43 +114,93 @@ module.exports = Schema.extend({
96
114
var model = this ;
97
115
wrapError ( this , options ) ;
98
116
99
- var parse = function ( doc , cb ) {
100
- model . parse ( doc ) ;
101
- cb ( null , doc ) ;
117
+ var success = options . success ;
118
+ options . success = function ( resp ) {
119
+ if ( success ) {
120
+ success ( model , resp , options ) ;
121
+ }
122
+ model . trigger ( 'sync' ) ;
102
123
} ;
124
+ var start = new Date ( ) ;
125
+ var timeAtFirstDoc ;
126
+ var erroredOnDocs = [ ] ;
103
127
104
- var docs = [ ] ;
128
+ // No results found
129
+ var onEmpty = function ( ) {
130
+ model . is_fetching = false ;
131
+ model . documents . reset ( ) ;
132
+ model . documents . trigger ( 'sync' ) ;
133
+ options . success ( { } ) ;
134
+ } ;
105
135
136
+ var docs = [ ] ;
106
137
var addToDocuments = function ( doc , cb ) {
107
138
docs . push ( doc ) ;
108
139
cb ( ) ;
109
140
} ;
110
141
142
+ var parse = function ( doc , cb ) {
143
+ if ( ! timeAtFirstDoc ) {
144
+ timeAtFirstDoc = new Date ( ) ;
145
+ }
146
+ try {
147
+ model . parse ( doc ) ;
148
+ } catch ( err ) {
149
+ erroredOnDocs . push ( doc ) ;
150
+ metrics . track ( 'Schema: Error: Parse' , err , {
151
+ doc : doc ,
152
+ schema : model . serialize ( )
153
+ } ) ;
154
+ }
155
+ cb ( null , doc ) ;
156
+ } ;
157
+
111
158
var onEnd = function ( err ) {
112
159
model . is_fetching = false ;
113
160
if ( err ) {
161
+ metrics . track ( 'Schema: Error: End' , err , {
162
+ schema : model
163
+ } ) ;
114
164
return options . error ( model , err ) ;
115
165
}
116
166
model . documents . reset ( docs ) ;
117
167
model . documents . trigger ( 'sync' ) ;
118
- options . success ( { } ) ;
119
- } ;
120
168
121
- var success = options . success ;
122
- options . success = function ( resp ) {
123
- if ( success ) {
124
- success ( model , resp , options ) ;
125
- }
126
- model . trigger ( 'sync' ) ;
169
+ // @note (imlucas): Any other metrics? Feedback on `Schema *`?
170
+ metrics . track ( 'Schema: Complete' , {
171
+ Duration : new Date ( ) - start ,
172
+ 'Total Document Count' : model . total ,
173
+ 'Document Count' : model . documents ,
174
+ 'Errored Document Count' : erroredOnDocs . length ,
175
+ 'Time to First Doc' : timeAtFirstDoc - start ,
176
+ 'Schema Height' : model . height , // # of top level keys
177
+ 'Schema Width' : model . width , // max nesting depth
178
+ 'Schema Sparsity' : model . sparsity // lots of fields missing or consistent
179
+ } ) ;
180
+ options . success ( { } ) ;
127
181
} ;
128
182
129
183
model . trigger ( 'request' , { } , { } , options ) ;
130
184
131
- debug ( 'creating sample stream' ) ;
132
- app . client . sample ( model . ns , options )
133
- . pipe ( es . map ( parse ) )
134
- . pipe ( es . map ( addToDocuments ) )
135
- . pipe ( es . wait ( onEnd ) ) ;
185
+ app . client . count ( model . ns , options , function ( err , count ) {
186
+ if ( err ) {
187
+ metrics . track ( 'Schema: Error: Count' , err , {
188
+ schema : model
189
+ } ) ;
190
+ return options . error ( model , err ) ;
191
+ }
192
+ debug ( 'options' , options , 'count' , count . count ) ;
193
+ model . total = count . count ;
194
+ if ( model . total === 0 ) {
195
+ return onEmpty ( ) ;
196
+ }
197
+
198
+ debug ( 'creating sample stream' ) ;
199
+ app . client . sample ( model . ns , options )
200
+ . pipe ( es . map ( parse ) )
201
+ . pipe ( es . map ( addToDocuments ) )
202
+ . pipe ( es . wait ( onEnd ) ) ;
203
+ } ) ;
136
204
} ,
137
205
serialize : function ( ) {
138
206
var res = this . getAttributes ( {
0 commit comments