Adding dataset images

Author: akariv

.dockerignore

@@ -22,3 +22,5 @@ LICENSE
 README.md
 tox.ini
 *.secret.env
+./data/
+node_modules/

Dockerfile

@@ -1,6 +1,8 @@
 FROM frictionlessdata/datapackage-pipelines:2.1.8
 
-RUN apk --update --no-cache add bash wget
+RUN apk --update --no-cache add bash wget nodejs npm nss chromium
+RUN npm install -g npm@latest
+RUN cd /pipelines/ && PUPPETEER_SKIP_CHROMIUM_DOWNLOAD="true" npm install puppeteer
 
 COPY docker-dpp-run.sh /dpp/docker/run.sh
 
@@ -12,6 +14,7 @@ COPY setup.py /pipelines/
 RUN python3 -m pip install -e .
 
 ENV DPP_ELASTICSEARCH=localhost:19200
+ENV CHROME_BIN="/usr/bin/chromium-browser"
 
 COPY datapackage_pipelines_migdar /pipelines/datapackage_pipelines_migdar
 COPY download_search_results_unique_records.sh /pipelines/

datapackage_pipelines_migdar/flows/dataset_assets.py

@@ -0,0 +1,29 @@
+import os
+import subprocess
+
+import dataflows as DF
+
+SCREENSHOT = os.path.join(os.path.dirname(__file__), 'node', 'screenshot.js')
+
+
+def do_screenshot():
+    def func(row):
+        doc_id = row['doc_id']
+        url = f'https://yodaat.org/card/{doc_id}'
+        outpath = os.path.join('data', os.path.dirname(doc_id))
+        os.makedirs(outpath, exist_ok=True)
+        outpath = os.path.join('data', doc_id + '.png')
+        subprocess.call(['node', SCREENSHOT, url, outpath, '.card'])
+    return func
+
+
+def flow(*_, path='data/datasets_in_es'):
+    return DF.Flow(
+        DF.load('{}/datapackage.json'.format(path)),
+        do_screenshot(),
+        DF.update_resource(-1, **{'dpp:streaming': True})
+    )
+
+
+if __name__ == '__main__':
+    flow(path='https://api.yodaat.org/data/datasets_in_es').process()

datapackage_pipelines_migdar/flows/node/.gitignore

@@ -0,0 +1 @@
+node_modules/

datapackage_pipelines_migdar/flows/node/dataset.png

@@ -0,0 +1,23 @@
+const puppeteer = require('puppeteer');
+
+(async () => {
+    const [url, filename, selector] = process.argv.slice(2);
+
+    const browser = await puppeteer.launch({
+        headless: true,
+        executablePath: process.env.CHROME_BIN || null,
+        args: ['--no-sandbox', '--headless', '--disable-gpu', '--disable-dev-shm-usage']
+    });
+    const page = await browser.newPage();
+    page.setViewport({width: 1300, height: 1200});
+    await page.goto(url);
+    await page.waitForSelector(selector + ' svg');
+
+    const rect = await page.evaluate(selector => {
+        const element = document.querySelector(selector);
+        const {x, y, width, height} = element.getBoundingClientRect();
+        return {x, y, width, height};
+    }, selector);
+    await page.screenshot({path: filename, clip: rect});
+    await browser.close();
+})();