Merge pull request #28 from autogram-is/0.6.0-dev

0.6.0 dev

Author: eaton

package.json

@@ -31,9 +31,9 @@
     "url": "https://autogram.is"
   },
   "contributors": [
-      "Jeff Eaton <jeff@autogram.is> (http://eaton.fyi)",
-      "Ethan Marcotte <karen@autogram.is> (https://ethanmarcotte.com)",
-      "Karen McGrane <karen@autogram.is> (https://karenmcgrane.com)"
+    "Jeff Eaton <jeff@autogram.is> (http://eaton.fyi)",
+    "Ethan Marcotte <karen@autogram.is> (https://ethanmarcotte.com)",
+    "Karen McGrane <karen@autogram.is> (https://karenmcgrane.com)"
   ],
   "license": "MIT",
   "repository": {
@@ -67,7 +67,6 @@
     "@oclif/plugin-plugins": "^2.1.6",
     "@sindresorhus/is": "^5.3.0",
     "@sindresorhus/slugify": "^2.1.1",
-    "@types/jsonld": "^1.5.8",
     "@vladfrangu/async_event_emitter": "^2.1.2",
     "arrify": "^3.0.0",
     "chalk": "^5.1.2",
@@ -90,6 +89,7 @@
     "open": "^8.4.0",
     "prepend-http": "^4.0.0",
     "protocolify": "^4.0.0",
+    "read-pkg-up": "^9.1.0",
     "reflect-metadata": "^0.1.13",
     "schema-dts": "^1.1.0",
     "terminal-link": "^3.0.0",
@@ -112,6 +112,7 @@
     "@types/node": "^18.11.0",
     "@types/object-hash": "^2.2.1",
     "@types/uuid": "^8.3.4",
+    "@types/jsonld": "^1.5.8",
     "oclif": "^3",
     "shx": "^0.3.3",
     "ts-node": "^10.8.0",

src/cli/commands/db/index.ts

@@ -24,17 +24,27 @@ export default class GraphInfo extends SgCommand {
 
     if (dbStatus) {
       await graph.db.listCollections()
-        .then(collections => collections.map(collection => {
-          return {
-            name: collection.name,
-            type: (collection.type === 2) ? 'document' : 'edge',
-          }
-        }))
+        .then(async metadata => {
+            const data: Record<string, unknown>[] = [];
+            for (let coll of metadata) {
+              const collection = graph.db.collection(coll.name);
+              const details = await collection.figures();
+              data.push({
+                name: coll.name,
+                type: (coll.type === 2) ? 'document' : 'edge',
+                records: details.count.toLocaleString(),
+                bytes: Number.parseInt(details.figures['documentsSize']).toLocaleString()
+              });
+            }
+            return data;
+        })
         .then(collections => {
           this.ux.table(collections, {
             name: { header: 'Collection', minWidth: 20 },
             type: {},
-          });
+            records: {},
+            bytes: {}
+          }, { sort: 'type' });
         })
     }
   }

src/cli/commands/project/ga.ts

@@ -17,17 +17,14 @@ export default class GA extends SgCommand {
     const {flags} = await this.parse(GA);
 
     // Obtain user credentials to use for the request
-    await GoogleTools.authenticate();
+    await GoogleTools.ServiceAccount.authenticate();
 
-    const res = await GoogleTools.analyticsReporting.reports.batchGet({
-      requestBody: {
-        reportRequests: [{
-          viewId: flags.view,
-          dateRanges: [ { startDate: '14daysAgo', endDate: '7daysAgo' } ],
-          metrics: [ { expression: 'ga:users', } ],
-        }],
-      },
+    const request = GoogleTools.UniversalAnalytics.buildUaRequest(flags.view, {
+      dimension: 'page',
+      pageSize: 10,
     });
-    console.log(res.data.reports?.pop()?.data);
+
+    const results = await GoogleTools.UniversalAnalytics.fetchUaReport(request);
+    this.ux.styledObject(results.data?.rows?.map(value => value.metrics));
   }
 }

src/cli/commands/screenshot.ts

@@ -31,6 +31,9 @@ export default class Screenshot extends SgCommand {
     selector: Flags.string({
       summary: 'CSS selector for element to capture',
     }),
+    limit: Flags.integer({
+      summary: 'Max number of matching elements to capture',
+    }),
     viewport: Flags.string({
       summary: 'Page viewport size',
       description: "Viewport can be a resolution in 'width,height' format; a preset name (iphone, ipad, hd, or fhd); or the preset 'all', which generates one screenshot for each defined preset.'",
@@ -64,20 +67,19 @@ export default class Screenshot extends SgCommand {
   }
 
   async run() {
-    const {project} = await this.getProjectContext();
     const {argv: urls, flags} = await this.parse(Screenshot);
 
     const captureTool = new ScreenshotTool();
     captureTool.on('capture', filename => this.ux.info(`Saved ${filename}...`));
 
     const options:Partial<ScreenshotOptions> = {
-      storage: project.files('storage'),
       directory: flags.directory,
       viewports: [flags.viewport],
       orientation: flags.orientation,
       selectors: flags.selector ? [flags.selector] : undefined,
       type: flags.format,
       fullPage: flags.fullpage,
+      limit: flags.limit ?? Infinity
     }
 
     const crawler = new PlaywrightCrawler({

src/cli/sg-command.ts

@@ -101,8 +101,8 @@ export abstract class SgCommand extends Command {
 
     try {
       const {flags} = await this.parse(this.constructor as typeof SgCommand);
-      const _configFilePath = is.string(flags.config) ? flags.config : undefined;
-      project = await Project.config({ _configFilePath });
+      const options = is.string(flags.config) ? { _configFilePath: flags.config } : undefined;
+      project = await Project.config(options);
       graph = await project.graph();
     } catch(error: unknown) {
       if (is.error(error)) {

src/model/helpers/unique-url-set.ts

@@ -24,7 +24,7 @@ export class UniqueUrlSet extends Set<UniqueUrl> {
     const uu = this.parse(value);
     if (uu) {
       super.add(uu);
-      this.verifier.add(uu.id);
+      this.verifier.add(uu.documentId);
     } else {
       this.unparsable.add(value as string);
     }
@@ -35,7 +35,7 @@ export class UniqueUrlSet extends Set<UniqueUrl> {
   override has(value: ValidUniqueUrlInput): boolean {
     const uu = this.parse(value);
     if (uu) {
-      return this.verifier.has(uu.id);
+      return this.verifier.has(uu.documentId);
     }
 
     return false;
@@ -44,9 +44,9 @@ export class UniqueUrlSet extends Set<UniqueUrl> {
   override delete(value: ValidUniqueUrlInput): boolean {
     const uu = this.parse(value);
     if (uu) {
-      this.verifier.delete(uu.id);
+      this.verifier.delete(uu.documentId);
       for (const u of this) {
-        if (u.id === uu.id) {
+        if (u.documentId === uu.documentId) {
           super.delete(u);
         }
       }

src/model/vertices/vertice.ts

@@ -44,7 +44,7 @@ export abstract class Vertice {
     } else if (is.array(r)) {
       return r.join('/');
     } else {
-      return r.id;
+      return r.documentId;
     }
   }
 
@@ -141,7 +141,7 @@ export abstract class Vertice {
     return this._key;
   }
 
-  get id(): string {
+  get documentId(): string {
     if (is.undefined(this._id)) {
       return [this._collection, this.key].join('/');
     }

src/services/arango-store.ts

@@ -182,13 +182,13 @@ export class ArangoStore {
     // When bulk deleting, remove edges first.
     for (const edge of input) {
       if (isEdge(edge)) {
-        promises.push(this.db.collection(edge._collection).remove(edge.id));
+        promises.push(this.db.collection(edge._collection).remove(edge.documentId));
       }
     }
 
     for (const vertice of input) {
       if (!isEdge(vertice)) {
-        promises.push(this.db.collection(vertice._collection).remove(vertice.id));
+        promises.push(this.db.collection(vertice._collection).remove(vertice.documentId));
       }
     }
 

src/spider/options.ts

@@ -4,6 +4,9 @@ import { mimeGroups } from './helpers/mime.js';
 import {SpiderHook} from './hooks/index.js';
 import {EnqueueUrlOptions} from './links/index.js';
 import {SpiderRequestHandler} from './handlers/index.js';
+import {readPackageUp} from 'read-pkg-up';
+
+const pkgData = await readPackageUp();
 
 export type SpiderOptions = InternalSpiderOptions & Omit<PlaywrightCrawlerOptions,
 'preNavigationHooks' |
@@ -165,4 +168,5 @@ const defaultSpiderOptions: InternalSpiderOptions = {
   urlOptions: {},
   parseMimeTypes: mimeGroups.page,
   downloadMimeTypes: [],
+  userAgent: `Spidergram ${pkgData?.packageJson.version}`
 };

src/tools/google/index.ts

@@ -1,3 +1,3 @@
-export * from './jwt-auth.js';
-export * from './universal-analytics.js';
-export * from './pagespeed.js';
+export * as ServiceAccount from './jwt-auth.js';
+export * as UniversalAnalytics from './universal-analytics.js';
+export * as PageSpeed from './pagespeed.js';