Merge pull request amplab#4 from shivaram/master

Set spark mem using slave memory

Author: mateiz

deploy_templates.py

@@ -11,9 +11,19 @@
 # ZooKeeper URL, etc as read from the environment.
 
 # Find system memory in KB and compute Spark's default limit from that
-system_ram_kb = int(
-  os.popen("cat /proc/meminfo | grep MemTotal | awk '{print $2}'")
-    .read().strip())
+mem_command = "cat /proc/meminfo | grep MemTotal | awk '{print $2}'"
+
+master_ram_kb = int(
+  os.popen(mem_command).read().strip())
+# This is the master's memory. Try to find slave's memory as well
+first_slave = os.popen("cat /root/spark-ec2/slaves | head -1").read().strip()
+
+slave_mem_command = "ssh -t -o StrictHostKeyChecking=no %s %s" %\
+        (first_slave, mem_command)
+slave_ram_kb = int(os.popen(slave_mem_command).read().strip())
+
+system_ram_kb = min(slave_ram_kb, master_ram_kb)
+
 system_ram_mb = system_ram_kb / 1024
 if system_ram_mb > 20*1024:
   # Leave 3 GB for the OS, HDFS and buffer cache