@@ -6,8 +6,24 @@ var filterableMixin = require('ampersand-collection-filterable');
66var SampledDocumentCollection = require ( './sampled-document-collection' ) ;
77var es = require ( 'event-stream' ) ;
88var debug = require ( 'debug' ) ( 'scout:models:schema' ) ;
9+ var debugMetrics = require ( 'debug' ) ( 'scout:metrics' ) ;
910var app = require ( 'ampersand-app' ) ;
1011
12+ // @todo : stub for metrics module. currently just logs debug messages with scout:metrics marker
13+ var metrics = {
14+ track : function ( label , err , obj ) {
15+ if ( ! obj ) {
16+ obj = err ;
17+ err = null ;
18+ }
19+ if ( err ) {
20+ debugMetrics ( 'metrics error: ' , err , obj ) ;
21+ } else {
22+ debugMetrics ( 'metrics log: ' , obj ) ;
23+ }
24+ }
25+ } ;
26+
1127/**
1228 * wrapping mongodb-schema's FieldCollection with a filterable mixin
1329 */
@@ -25,6 +41,8 @@ module.exports = Schema.extend({
2541 }
2642 } ,
2743 session : {
44+ // total number of documents counted under the given query
45+ total : 'number' ,
2846 is_fetching : {
2947 type : 'boolean' ,
3048 default : false
@@ -96,43 +114,93 @@ module.exports = Schema.extend({
96114 var model = this ;
97115 wrapError ( this , options ) ;
98116
99- var parse = function ( doc , cb ) {
100- model . parse ( doc ) ;
101- cb ( null , doc ) ;
117+ var success = options . success ;
118+ options . success = function ( resp ) {
119+ if ( success ) {
120+ success ( model , resp , options ) ;
121+ }
122+ model . trigger ( 'sync' ) ;
102123 } ;
124+ var start = new Date ( ) ;
125+ var timeAtFirstDoc ;
126+ var erroredOnDocs = [ ] ;
103127
104- var docs = [ ] ;
128+ // No results found
129+ var onEmpty = function ( ) {
130+ model . is_fetching = false ;
131+ model . documents . reset ( ) ;
132+ model . documents . trigger ( 'sync' ) ;
133+ options . success ( { } ) ;
134+ } ;
105135
136+ var docs = [ ] ;
106137 var addToDocuments = function ( doc , cb ) {
107138 docs . push ( doc ) ;
108139 cb ( ) ;
109140 } ;
110141
142+ var parse = function ( doc , cb ) {
143+ if ( ! timeAtFirstDoc ) {
144+ timeAtFirstDoc = new Date ( ) ;
145+ }
146+ try {
147+ model . parse ( doc ) ;
148+ } catch ( err ) {
149+ erroredOnDocs . push ( doc ) ;
150+ metrics . track ( 'Schema: Error: Parse' , err , {
151+ doc : doc ,
152+ schema : model . serialize ( )
153+ } ) ;
154+ }
155+ cb ( null , doc ) ;
156+ } ;
157+
111158 var onEnd = function ( err ) {
112159 model . is_fetching = false ;
113160 if ( err ) {
161+ metrics . track ( 'Schema: Error: End' , err , {
162+ schema : model
163+ } ) ;
114164 return options . error ( model , err ) ;
115165 }
116166 model . documents . reset ( docs ) ;
117167 model . documents . trigger ( 'sync' ) ;
118- options . success ( { } ) ;
119- } ;
120168
121- var success = options . success ;
122- options . success = function ( resp ) {
123- if ( success ) {
124- success ( model , resp , options ) ;
125- }
126- model . trigger ( 'sync' ) ;
169+ // @note (imlucas): Any other metrics? Feedback on `Schema *`?
170+ metrics . track ( 'Schema: Complete' , {
171+ Duration : new Date ( ) - start ,
172+ 'Total Document Count' : model . total ,
173+ 'Document Count' : model . documents ,
174+ 'Errored Document Count' : erroredOnDocs . length ,
175+ 'Time to First Doc' : timeAtFirstDoc - start ,
176+ 'Schema Height' : model . height , // # of top level keys
177+ 'Schema Width' : model . width , // max nesting depth
178+ 'Schema Sparsity' : model . sparsity // lots of fields missing or consistent
179+ } ) ;
180+ options . success ( { } ) ;
127181 } ;
128182
129183 model . trigger ( 'request' , { } , { } , options ) ;
130184
131- debug ( 'creating sample stream' ) ;
132- app . client . sample ( model . ns , options )
133- . pipe ( es . map ( parse ) )
134- . pipe ( es . map ( addToDocuments ) )
135- . pipe ( es . wait ( onEnd ) ) ;
185+ app . client . count ( model . ns , options , function ( err , count ) {
186+ if ( err ) {
187+ metrics . track ( 'Schema: Error: Count' , err , {
188+ schema : model
189+ } ) ;
190+ return options . error ( model , err ) ;
191+ }
192+ debug ( 'options' , options , 'count' , count . count ) ;
193+ model . total = count . count ;
194+ if ( model . total === 0 ) {
195+ return onEmpty ( ) ;
196+ }
197+
198+ debug ( 'creating sample stream' ) ;
199+ app . client . sample ( model . ns , options )
200+ . pipe ( es . map ( parse ) )
201+ . pipe ( es . map ( addToDocuments ) )
202+ . pipe ( es . wait ( onEnd ) ) ;
203+ } ) ;
136204 } ,
137205 serialize : function ( ) {
138206 var res = this . getAttributes ( {
0 commit comments