Live preview for spark script samples

Author: luisbelloch

spark/.gitignore

@@ -1,2 +1,3 @@
 _work.py
+live.html
 

spark/live.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from pyspark.sql import SparkSession
+
+spark = SparkSession.builder.master("local").appName("SQL").getOrCreate()
+df = spark.read.load('data/containers_tiny.parquet')
+df.select("ship_imo", "container_id", "net_weight").show()
+

spark/live.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+set -euo pipefail
+
+fswatch live.py | while read -r fpath; do \
+    echo -e "\033[0;36mRELOAD\033[0m $fpath $(date +"%H%m%S")"
+    pygmentize -f html -O full,linenos=1 -o live.html live.py
+    scp live.html [email protected]:/var/www/html/index.html
+done
+