barryclark · 93tilinfinity · Jan 23, 2021 · Jan 23, 2021 · Jan 23, 2021 · Jan 23, 2021
diff --git a/CNAME b/CNAME
@@ -1 +1 @@
-
+www.neilchandarana.com
diff --git a/_config.yml b/_config.yml
@@ -3,13 +3,13 @@
 #
 
 # Name of your site (displayed in the header)
-name: Your Name
+name: Neil Chandarana
 
 # Short bio or description (displayed in the header)
-description: Web Developer from Somewhere
+description: A home for poorly researched ideas on data systems
 
 # URL of your avatar or profile pic (you could use your GitHub profile pic)
-avatar: https://raw.githubusercontent.com/barryclark/jekyll-now/master/images/jekyll-logo.png
+avatar: #https://raw.githubusercontent.com/barryclark/jekyll-now/master/images/jekyll-logo.png
 
 #
 # Flags below are optional
@@ -18,17 +18,17 @@ avatar: https://raw.githubusercontent.com/barryclark/jekyll-now/master/images/je
 # Includes an icon in the footer for each username you enter
 footer-links:
   dribbble:
-  email:
+  email: [email protected]
   facebook:
   flickr:
-  github: barryclark/jekyll-now
-  instagram:
+  github: 93tilinfinity
+  instagram: neilchanda
   linkedin:
   pinterest:
   rss: # just type anything here for a working RSS icon
-  twitter: jekyllrb
+  twitter: 
   stackoverflow: # your stackoverflow profile, e.g. "users/50476/bart-kiers"
-  youtube: # channel/<your_long_string> or user/<user-name>
+  youtube: channel/UCf05rA2fSzzL1H_RrZ7LN7w # channel/<your_long_string> or user/<user-name>
   googleplus: # anything in your profile username that comes after plus.google.com/
 
 
@@ -41,7 +41,7 @@ google_analytics:
 
 # Your website URL (e.g. http://barryclark.github.io or http://www.barryclark.co)
 # Used for Sitemap.xml and your RSS feed
-url:
+url: # http://neilchandarana.com
 
 # If you're hosting your site at a Project repository on GitHub pages
 # (http://yourusername.github.io/repository-name)
@@ -76,6 +76,10 @@ sass:
 gems:
   - jekyll-sitemap # Create a sitemap using the official Jekyll sitemap gem
   - jekyll-feed # Create an Atom feed using the official Jekyll feed gem
+  - jemoji
+
+plugins:
+  - jemoji
 
 # Exclude these files from your production _site
 exclude:

diff --git a/_includes/collecttags.html b/_includes/collecttags.html
@@ -0,0 +1,23 @@
+{% assign rawtags = "" %}
+{% for post in site.posts %}
+    {% assign ttags = post.tags | join:'|' | append:'|' %}
+    {% assign rawtags = rawtags | append:ttags %}
+{% endfor %}
+{% assign rawtags = rawtags | split:'|' | sort %}
+
+{% assign site.tags = "" %}
+{% for tag in rawtags %}
+    {% if tag != "" %}
+        {% if tags == "" %}
+            {% assign tags = tag | split:'|' | sort %}
+        {% endif %}
+        {% unless tags contains tag %}
+            {% assign tags = tags | join:'|' | append:'|' | append:tag | split:'|' %}
+        {% endunless %}
+    {% endif %}
+{% endfor %}
+
+
+
+
+
diff --git a/_includes/tagcloud.html b/_includes/tagcloud.html
@@ -0,0 +1,12 @@
+{% capture temptags %}
+  {% for tag in site.tags %}
+    {{ tag[1].size | plus: 1000 }}#{{ tag[0] }}#{{ tag[1].size }}
+  {% endfor %}
+{% endcapture %}
+{% assign sortedtemptags = temptags | split:' ' | sort | reverse %}
+Tags:
+{% for temptag in sortedtemptags %}
+  {% assign tagitems = temptag | split: '#' %}
+  {% capture tagname %}{{ tagitems[1] }}{% endcapture %}
+  [<a href="/tag/{{ tagname }}"><code style="color:#a94064"><nobr>#{{ tagname }}</nobr></code></a>]
+{% endfor %}
diff --git a/_layouts/default.html b/_layouts/default.html
@@ -2,39 +2,45 @@
 <html>
   <head>
     <title>{% if page.title %}{{ page.title }} – {% endif %}{{ site.name }} – {{ site.description }}</title>
-
     {% include meta.html %}
-
     <!--[if lt IE 9]>
       <script src="http://html5shiv.googlecode.com/svn/trunk/html5.js"></script>
     <![endif]-->
-
     <link rel="stylesheet" type="text/css" href="{{ site.baseurl }}/style.css" />
     <link rel="alternate" type="application/rss+xml" title="{{ site.name }} - {{ site.description }}" href="{{ site.baseurl }}/feed.xml" />
-
-    <!-- Created with Jekyll Now - http://github.com/barryclark/jekyll-now -->
+    {% include collecttags.html %}
   </head>
 
   <body>
     <div class="wrapper-masthead">
       <div class="container">
         <header class="masthead clearfix">
-          <a href="{{ site.baseurl }}/" class="site-avatar"><img src="{{ site.avatar }}" /></a>
-
+<!--          <a href="{{ site.baseurl }}/" class="site-avatar"><img src="/images/logo-removebg-preview.png" /></a>-->
           <div class="site-info">
+<!--            <h1>-->
+<!--              <a href="{{ site.baseurl }}/">-->
+<!--                <img src="/images/logo-removebg-preview.png" alt="Neil Chandarana logo" />-->
+<!--              </a>-->
+<!--            </h1>-->
             <h1 class="site-name"><a href="{{ site.baseurl }}/">{{ site.name }}</a></h1>
             <p class="site-description">{{ site.description }}</p>
           </div>
 
           <nav>
             <a href="{{ site.baseurl }}/">Blog</a>
-            <a href="{{ site.baseurl }}/about">About</a>
+<!--            <a href="{{ site.baseurl }}/links/">Links</a>-->
+            <a href="{{ site.baseurl }}/bio/">Bio</a>
           </nav>
         </header>
       </div>
     </div>
 
     <div id="main" role="main" class="container">
+<!--        {% if page.url == "/" %}-->
+<!--        <div class="tagcloud">-->
+<!--            {% include tagcloud.html %}-->
+<!--        </div>-->
+<!--        {% endif %}-->
       {{ content }}
     </div>
 

diff --git a/_layouts/post.html b/_layouts/post.html
@@ -5,12 +5,23 @@
 <article class="post">
   <h1>{{ page.title }}</h1>
 
-  <div class="entry">
-    {{ content }}
+  <div class="post_container">
+
+    <div class="meta1">
+    {{ page.date | date: "%B %e, %Y" }}
+    </div>
+    <div class="meta2">
+    {% for tag in page.tags %}
+      {% capture tag_name %}{{ tag }}{% endcapture %}
+<!--      <a href="/tag/{{ tag_name }}"><code style="color:#a94064"><nobr>#{{ tag_name }}</nobr></code>&nbsp;</a>-->
+      <code style="color:#a94064"><nobr>#{{ tag_name }}</nobr></code>&nbsp;</a>
+    {% endfor %}
+    </div>
+
   </div>
 
-  <div class="date">
-    Written on {{ page.date | date: "%B %e, %Y" }}
+  <div class="entry">
+    {{ content }}
   </div>
 
   {% include disqus.html %}

diff --git a/_posts/2014-3-3-Hello-World.md b/_posts/2014-3-3-Hello-World.md
diff --git a/_posts/2021-1-23-a-paradigm-for-life.md b/_posts/2021-1-23-a-paradigm-for-life.md
@@ -0,0 +1,49 @@
+---
+layout: post
+title: 'Input-> System-> Output: A paradigm for life'
+tags: thinking
+---
+I came across an article that claimed a piece of analysis failed because of ‘bad’ data. 
+
+It reminded me of a phrase I used to hear on the trading floor, “US non-farm payrolls beat estimates”.
+
+Is it possible for data to be inherently good or bad? I am not convinced. The state of data should only be that they 
+exist, and are held as the truth because they exist. In other words, US non-farm payrolls didn't beat estimates, your 
+estimate was too low.
+
+There must be another reason to explain the system’s unexpected performance. Consider the following paradigm.
+
+![input,system,output](/images/blog_01_2021/PXL_20210122_185257907~2.jpg)
+
+When applied to a machine learning or data science project might give something like the following 4 steps.
+
+![ML system flow](/images/blog_01_2021/PXL_20210125_144448228~3.jpg)
+
+Putting my paranoid/Trader cap on, 4 steps mean 4 modes of failure. 
+
+The rest of this blog shall pay close attention to mode 2. Otherwise labelled as DevOps, data architecture, data engineering.
+ I never know which one to use so I’ll stick with ‘move data from there to here’.
+
+The aim of this blog is to challenge myself and hopefully the reader to think about big data systems at a layer of 
+abstraction that remains invariant in time.
+
+Learn to fish and you’ll never be hungry, or something like that.
+
+Stick around and you can expect to explore:
+* Database principles
+* Types of failure
+* Lambda architecture
+* Where today’s tools fit in
+* Lots of hand-drawn diagrams...no one does them anymore
+
+<!---
+
+layout: post
+title: You're up and running!
+
+Next you can update your site name, avatar and other options using the _config.yml file in the root of your repository (shown below).
+
+![_config.yml]({{ site.baseurl }}/images/config.png)
+
+The easiest way to make your first post is to edit this one. Go into /_posts/ and update the Hello World markdown file. For more instructions head over to the [Jekyll Now repository](https://github.com/barryclark/jekyll-now) on GitHub.
+--->
diff --git a/_posts/2021-1-24-scaling-relational-databases-queues.md b/_posts/2021-1-24-scaling-relational-databases-queues.md
@@ -0,0 +1,38 @@
+---
+layout: post
+title: 'Queueing a relational database'
+tags: relationaldatabase
+---
+
+Neil’s notes: 
+* Hitting the database directly is bad
+* Adding a queue helps
+* Batching updates helps
+
+Suppose you decide to build a simple web analytics application. 
+
+The application you have in mind tracks the number of pageviews for any URL a customer wishes to track. Additionally, 
+the application should be able to report the top 100 URLs by number of pageviews.
+
+You start with the following data schema:
+
+![Data schema](/images/blog_01_2021/PXL_20210125_145337225~2.jpg)
+
+You build the back end to consist of a Relational Database Management System (say MySQL), a table with the above schema 
+and a web server. Each time a web page tracked by your application is loaded, the web page pings your web server with
+ the pageview, and your web server increments the corresponding row in the database. 
+
+![A web application](/images/blog_01_2021/PXL_20210125_145356099~2.jpg)
+
+The web application is a success, and traffic is growing like wildfire. You start getting lots of emails from your monitoring systems all with the same error: 
+
+“Timeout error on inserting to the database”
+
+Your logs tell you that write requests are timing out because the database cannot keep up with the load. Your system is losing valuable information and you need to act quickly.
+
+You realise that it is inefficient to perform a single increment per request. You draw up and implement a queuing system between the web server and the database so whenever a pageview is received, that event is added to the queue. You also create a worker that reads 100 events at a time off the queue and batches them into a single update request to your database. 
+
+![A web application with queue](/images/blog_01_2021/PXL_20210125_145409766~2.jpg)
+
+It works well and the web server no longer pings the database directly and you no longer receive timeout notifications!
+
diff --git a/_posts/2021-1-25-scaling-relational-databases-sharding.md b/_posts/2021-1-25-scaling-relational-databases-sharding.md
@@ -0,0 +1,33 @@
+---
+layout: post
+title: 'Sharding a relational database'
+tags: relationaldatabase
+---
+
+Neil’s notes:
+* Queues can’t escape overload, the database is the bottleneck
+* Spread the write load horizontally across multiple machines (sharding)
+* Sharding requires significant day 1 work
+* Re-sharding cost is very real
+
+![Sharded Application](/images/blog_01_2021/PXL_20210125_150212123~2.jpg)
+
+Your application goes viral. Your worker can’t keep up with the writes so you add more workers and parallelise
+ the updates. This works for a short time but you realise the bottleneck is in the database.
+
+A Google search on scaling write-heavy brings up Sharding or horizontal partitioning. To spread the write load 
+by partitioning the table across multiple servers using a shard key.
+
+How to shard:
+* Choose a hash function
+* Shard_id = hash(key) mod n_shards
+
+You write a script to map all the rows of your table to you hash function and split the table into 4 shards. 
+It takes a while to run and you don’t want new pageview increments to unbalance your uniform partition. 
+You decide turn off the worker till partition completion caching the interim pageview increments. 
+
+Your application now needs to know how to find the shard for each key so you wrap a library around your 
+database handling code that reads the number of shards from a new configuration file you created. 
+
+You also write additional logic to query the top 100 URLs from each shard and merge them to return a global 
+top 100 URLs.
diff --git a/_posts/2021-1-27-scaling-relational-databases-issues.md b/_posts/2021-1-27-scaling-relational-databases-issues.md
@@ -0,0 +1,29 @@
+---
+layout: post
+title: "Mo' data, mo' problems"
+tags: relationaldatabase
+---
+
+Neil’s notes:
+* Bugs inevitably make it to production code
+* Fault-tolerance might be more important than I initially thought
+* Complexity comes in more than one form; setting up a system + operating it + ?
+
+_“I used to build cool features for customers. Now I’m spending all my time dealing with problems reading and writing data”._
+
+As the application becomes more and more popular, you keep having to re-shard the database into more and more shards to keep up with the write load. Each re-shard, gets more and more painful because there’s so many more shards + active workers + config changes to co-ordinate. 
+
+You forget to update the config file and that causes pageview increments to be written to the wrong shards. You have to write a one-off script to manually comb through the data and fix the incorrect increments.
+
+Eventually, there are so many shards across so many servers that occasionally the disk on one of your servers goes bad. While that machine is down, the data on its shard are unavailable to your application. You do a couple things to address this:
+
+* You set a pending queue to divert increments for an unavailable shard. That pending queue flushes every 5 minutes. 
+* You backup each shard without write ability. Users can now still use the application when the master shard goes down. 
+
+Job done? Not quite.
+
+While working on your queue/worker code, you notice you accidently deployed a bug that increments pageview count by 3 instead of by 1. There is no way of knowing which data got corrupted so your backups don’t help.
+
+After all your efforts to make the system tolerant to machine faults and scalable, there is still no resilience to human faults. 
+
+Unfortunately, as the system complexity increases the fault probability increases. On top of that, additional work is pushed to you both in operating the database and developing the application code.
diff --git a/_posts/2021-1-28-data-vs-information.md b/_posts/2021-1-28-data-vs-information.md
@@ -0,0 +1,39 @@
+---
+layout: post
+title: 'Is there a difference between Data and Information?'
+tags: thinking
+---
+
+**TL;DR:** Yes.
+
+In the last few posts, I jumped straight into the pitfalls of old school databases. It’s not profound or original material and more practically serves as a history lesson in data systems. I never enjoyed how history was taught at school, looking back I believe it was because I never had any concept of a particular event’s interconnectedness with other events around it in both time and geography. 
+
+Sharing this blog with a few friends, the feedback was more or less _“blog goes over my head but I like the bio”_. That’s no fun. 
+
+This blog is about data systems. To give a flavour of interconnectedness, data systems are the backbone of almost every application used today. They are like the sewage system; undoubtably important but for most, rarely attracts thought. 
+
+**What is a data system?**
+
+A data system answers questions based on information acquired in the past up to the present.
+
+A bank account answers: What is my current balance? What transactions did I make in January?
+
+A social media network answers: How many followers do I have? What is this person’s name?
+
+Data systems don’t just memorise + regurgitate, they combine different bits and pieces of information together to produce answers. For example, your current balance is combines information on all transactions on your account. 
+
+**What bits and pieces do data systems store?** 
+
+A crucial observation is that some information can be derived from other pieces of information. 
+
+Your follower list is derived from the sequence of every follow and unfollow action each and every user has ever made on your profile. 
+
+You can keep tracing information back till eventually you get information that is not derived from anything. This what I’ll call data.
+
+**Data vs information: Is there a difference?**
+
+Unfortunately, the words data and information are often used interchangeably in media or within industry. Don’t accept poor word choice. Data are small block of truth that alone may have no meaning but within the right context and combined in the right way become information that does have meaning. Viewing information as a big object made up of irreducible blocks of data gives clarity on what data are and are not.
+
+Wrapping up, data and information are not the same. Anything you could ever imagine doing with data can be expressed as a function that takes in data and returns answers. So, the crux of everything you can learn about data systems is the following general-purpose definition:
+
+Query = function(all data)