File tree Expand file tree Collapse file tree 2 files changed +11
-3
lines changed
Expand file tree Collapse file tree 2 files changed +11
-3
lines changed Original file line number Diff line number Diff line change @@ -71,12 +71,14 @@ function getAllLinks(realMaxPagePromise) {
7171 getPageLinks ( currentPage , callback ) ;
7272 } else {
7373 pageslinks = concatToOneArray ( pageslinks ) ;
74+ console . log ( '\x1b[36m%s\x1b[0m' , '\n SUCCESS : all item(s) links crawled.' ) ;
75+ console . log ( '\x1b[36m%s\x1b[0m' , '\n START of item(s) crawling.' ) ;
7476 getItems ( pageslinks ) ;
7577 }
7678 }
7779 getPageLinks ( currentPage , callback ) ;
7880 } else {
79- console . log ( '\x1b[33m %s\x1b[0m' , '\n /!\\ Max page of this category is ' + realMaxPage + ' so ' + cmdResponse . pages + ' is to much :(' ) ;
81+ console . log ( '\x1b[31m %s\x1b[0m' , '\n /!\\ Max page of this category is ' + realMaxPage + ' so ' + cmdResponse . pages + ' is to much :(' ) ;
8082 process . exit ( ) ;
8183 }
8284
@@ -124,7 +126,7 @@ function getItems(pageslinks) {
124126 itemCategory = itemCategory . replace ( / / g, '' ) ;
125127 fsPath . writeFile ( './data/' + itemCategory + '.json' , JSON . stringify ( items ) , function ( err ) {
126128 if ( err ) console . log ( err ) ;
127- console . log ( '\x1b[33m %s\x1b[0m' , '\n SUCCESS : ' + pageslinks . length + ' item(s) were crawled.' ) ;
129+ console . log ( '\x1b[32m %s\x1b[0m' , '\n SUCCESS : ' + pageslinks . length + ' item(s) were crawled.' ) ;
128130 console . log ( '\x1b[33m%s\x1b[0m' , 'File ' + itemCategory + '.json' + ' was generated under "data/" folder.' ) ;
129131 process . exit ( ) ;
130132 } ) ;
Original file line number Diff line number Diff line change @@ -12,18 +12,24 @@ var CLI = require('clui'),
1212 Progress = CLI . Progress ;
1313
1414var currentPosition = 0 ;
15+ var progressbarPosition = 0 ;
1516var itemsList = [ ] ;
1617var thisProgressBar = new Progress ( 20 ) ;
1718
1819var getItems = exports . getItems = function ( links , back ) {
1920 getData ( links [ currentPosition ] , back , function ( item , back ) {
2021 itemsList . push ( item ) ;
21- console . log ( thisProgressBar . update ( currentPosition , links . length ) ) ;
22+ if ( progressbarPosition >= links . length * 0.05 ) {
23+ progressbarPosition = 0 ;
24+ console . log ( thisProgressBar . update ( currentPosition , links . length ) ) ;
25+ }
26+ progressbarPosition ++ ;
2227 currentPosition ++ ;
2328 // any more items in array?
2429 if ( currentPosition < links . length ) {
2530 getItems ( links , back ) ;
2631 } else {
32+ console . log ( thisProgressBar . update ( 100 , 100 ) ) ;
2733 back ( itemsList ) ;
2834 }
2935 } ) ;
You can’t perform that action at this time.
0 commit comments