Merge branch 'master' into tbb

Author: poojanilangekar

.gitignore

@@ -306,6 +306,7 @@ docs/dev
 # Protobuf files
 *.pb.cc
 *.pb.h
+*.pb
 
 # Third party
 third_party/gflags/

.travis.yml

@@ -1,4 +1,5 @@
 language: c++
+python: 3.6
 matrix:
     fast_finish: true
 
@@ -141,7 +142,7 @@ install:
 
 before_script:
     # first, run source_validator
-    - python ./script/validators/source_validator.py
+    - python script/validators/source_validator.py
 
 # build peloton (override this value to execute tests)
 script:

Jenkinsfile

@@ -36,6 +36,12 @@ include(cmake/CapnProtoMacros.cmake)
 # ---[ Google-protobuf
 include(cmake/ProtoBuf.cmake)
 
+# --[ tensorflow
+find_library(TFlowC
+        NAMES tensorflow
+        PATHS "/usr/local/lib")
+list(APPEND Peloton_LINKER_LIBS ${TFlowC})
+
 # ---[ Libevent
 find_package(Libevent REQUIRED)
 include_directories(SYSTEM ${LIBEVENT_INCLUDE_DIRS})

cmake/Dependencies.cmake

@@ -1,7 +1,6 @@
 # Finds Google Protocol Buffers library and compilers and extends
 # the standard cmake script with version and python generation support
-
-find_package( Protobuf REQUIRED )
+find_package( Protobuf REQUIRED)
 include_directories(SYSTEM ${PROTOBUF_INCLUDE_DIR})
 list(APPEND Peloton_LINKER_LIBS ${PROTOBUF_LIBRARIES})
 

cmake/ProtoBuf.cmake

@@ -1,6 +1,6 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # encoding: utf-8
-
+"""Format the ill-formatted code."""
 ## ==============================================
 ## GOAL : Format code, Update headers
 ## ==============================================
@@ -12,8 +12,15 @@
 import sys
 import datetime
 import subprocess
-import distutils.spawn
 
+from functools import reduce
+
+# Following is done so that we can import from ../helpers.py
+sys.path.append(
+    os.path.abspath(os.path.dirname(__file__)).replace('/formatting', '')
+    )
+from helpers import CLANG_FORMAT, PELOTON_DIR, CLANG_FORMAT_FILE, LOG,\
+clang_format
 
 ## ==============================================
 ## CONFIGURATION
@@ -22,8 +29,6 @@
 # NOTE: absolute path to peloton directory is calculated from current directory
 # directory structure: peloton/scripts/formatting/<this_file>
 # PELOTON_DIR needs to be redefined if the directory structure is changed
-CODE_SOURCE_DIR = os.path.abspath(os.path.dirname(__file__))
-PELOTON_DIR = reduce(os.path.join, [CODE_SOURCE_DIR, os.path.pardir, os.path.pardir])
 
 #other directory paths used are relative to peloton_dir
 PELOTON_SRC_DIR = os.path.join(PELOTON_DIR, "src")
@@ -34,15 +39,12 @@
 DEFAULT_DIRS.append(PELOTON_SRC_DIR)
 DEFAULT_DIRS.append(PELOTON_TESTS_DIR)
 
-CLANG_FORMAT = None
-CLANG_FORMAT_FILE = os.path.join(PELOTON_DIR, ".clang-format")
-
 ## ==============================================
 ##             HEADER CONFIGURATION
 ## ==============================================
 
 #header framework, dynamic information will be added inside function
-header_comment_line_1 = "//===----------------------------------------------------------------------===//\n"
+header_comment_line_1 =  "//===----------------------------------------------------------------------===//\n"
 header_comment_line_1 += "//\n"
 header_comment_line_1 += "//                         Peloton\n"
 header_comment_line_2  = "//\n"
@@ -56,81 +58,61 @@
 
 header_comment_1 = header_comment_line_1 + header_comment_line_2
 header_comment_3 = header_comment_line_4
-header_comment_5 = header_comment_line_6 + header_comment_line_7 + header_comment_line_8 \
-                        + header_comment_line_9
+header_comment_5 = header_comment_line_6 + header_comment_line_7 \
++ header_comment_line_8 + header_comment_line_9
 
 #regular expresseion used to track header
-header_regex = re.compile("((\/\/===-*===\/\/\n(\/\/.*\n)*\/\/===-*===\/\/[\n]*)\n\n)*")
-
-## ==============================================
-##             LOGGING CONFIGURATION
-## ==============================================
-
-LOG = logging.getLogger(__name__)
-LOG_handler = logging.StreamHandler()
-LOG_formatter = logging.Formatter(
-    fmt='%(asctime)s [%(funcName)s:%(lineno)03d] %(levelname)-5s: %(message)s',
-    datefmt='%m-%d-%Y %H:%M:%S'
-)
-LOG_handler.setFormatter(LOG_formatter)
-LOG.addHandler(LOG_handler)
-LOG.setLevel(logging.INFO)
+HEADER_REGEX = re.compile(r"((\/\/===-*===\/\/\n(\/\/.*\n)*\/\/===-*===\/\/[\n]*)\n\n)*")
 
 ## ==============================================
 ##           UTILITY FUNCTION DEFINITIONS
 ## ==============================================
 
-#format the file passed as argument
-def format_file(file_path, update_header, clang_format_code):
 
+def format_file(file_path, update_header, clang_format_code):
+    """Formats the file passed as argument."""
     file_name = os.path.basename(file_path)
     abs_path = os.path.abspath(file_path)
-    rel_path_from_peloton_dir = os.path.relpath(abs_path,PELOTON_DIR)
+    rel_path_from_peloton_dir = os.path.relpath(abs_path, PELOTON_DIR)
 
-    with open(file_path, "r+") as fd:
-        file_data = fd.read()
+    with open(file_path, "r+") as file:
+        file_data = file.read()
 
         if update_header:
             # strip old header if it exists
-            header_match = header_regex.match(file_data)
+            header_match = HEADER_REGEX.match(file_data)
             if not header_match is None:
-              LOG.info("Strip header from %s", file_name)
-              header_comment = header_match.group()
-              LOG.debug("Header comment : %s", header_comment)
-              file_data = file_data.replace(header_comment,"")
-            
+                LOG.info("Strip header from %s", file_name)
+                header_comment = header_match.group()
+                LOG.debug("Header comment : %s", header_comment)
+                file_data = file_data.replace(header_comment,"")
+
             # add new header
             LOG.info("Add header to %s", file_name)
             header_comment_2 = header_comment_line_3 + file_name + "\n"
-            header_comment_4 = header_comment_line_5 + rel_path_from_peloton_dir + "\n"
-            header_comment = header_comment_1 + header_comment_2 + header_comment_3 \
-                        + header_comment_4 + header_comment_5
+            header_comment_4 = header_comment_line_5\
+            + rel_path_from_peloton_dir + "\n"
+            header_comment = header_comment_1 + header_comment_2 \
+            + header_comment_3 + header_comment_4 \
+            + header_comment_5
             #print header_comment
 
             file_data = header_comment + file_data
 
-            fd.seek(0,0)
-            fd.truncate()
-            fd.write(file_data)
+            file.seek(0, 0)
+            file.truncate()
+            file.write(file_data)
 
         elif clang_format_code:
-            if CLANG_FORMAT is None:
-                LOG.error("clang-format seems not installed")
-                exit("clang-format seems not installed")
-            
-            formatting_command = CLANG_FORMAT + " -style=file -i " + file_path
-            LOG.info(formatting_command)
-            subprocess.call([CLANG_FORMAT, "-style=file", "-i", file_path])
+            clang_format(file_path)
 
     #END WITH
-
-    fd.close()
 #END FORMAT__FILE(FILE_NAME)
 
 
-#format all the files in the dir passed as argument
 def format_dir(dir_path, update_header, clang_format_code):
-    for subdir, dirs, files in os.walk(dir_path):
+    """Formats all the files in the dir passed as argument."""
+    for subdir, _, files in os.walk(dir_path):  # _ is for directories.
         for file in files:
             #print os.path.join(subdir, file)
             file_path = subdir + os.path.sep + file
@@ -142,53 +124,66 @@ def format_dir(dir_path, update_header, clang_format_code):
     #END FOR [os.walk]
 #END ADD_HEADERS_DIR(DIR_PATH)
 
-#find clang-format executable
-def find_clangformat():
-    global CLANG_FORMAT
-    #check for possible clang-format versions
-    for exe in ["clang-format", "clang-format-3.6", "clang-format-3.7", "clang-format-3.8"]:
-        path = distutils.spawn.find_executable(exe)
-        if not path is None:
-            CLANG_FORMAT = path
-            return
-
 
 ## ==============================================
 ##                 Main Function
 ## ==============================================
 
 if __name__ == '__main__':
 
-    parser = argparse.ArgumentParser(description='Update headers and/or format source code')
-
-    parser.add_argument("-u", "--update-header", help='Action: Update existing headers or add new ones', action='store_true')
-    parser.add_argument("-c", "--clang-format-code", help='Action: Apply clang-format to source code', action='store_true')
-    parser.add_argument("-f", "--staged-files", help='Action: Apply the selected action(s) to all staged files (git)', action='store_true')
-    parser.add_argument('paths', metavar='PATH', type=str, nargs='*',
-                        help='Files or directories to (recursively) apply the actions to')
-    
-    args = parser.parse_args()
-    
-    find_clangformat()
-
-    if args.staged_files:
-        targets = [os.path.abspath(os.path.join(PELOTON_DIR, f)) for f in subprocess.check_output(["git", "diff", "--name-only", "HEAD", "--cached", "--diff-filter=d"]).split()]
-        if not targets:
-            LOG.error("no staged files or not calling from a repository -- exiting")
+    PARSER = argparse.ArgumentParser(
+        description='Update headers and/or format source code'
+        )
+
+    PARSER.add_argument(
+        "-u", "--update-header",
+        help='Action: Update existing headers or add new ones',
+        action='store_true'
+        )
+    PARSER.add_argument(
+        "-c", "--clang-format-code",
+        help='Action: Apply clang-format to source code',
+        action='store_true'
+        )
+    PARSER.add_argument(
+        "-f", "--staged-files",
+        help='Action: Apply the selected action(s) to all staged files (git)',
+        action='store_true'
+        )
+    PARSER.add_argument(
+        'paths', metavar='PATH', type=str, nargs='*',
+        help='Files or directories to (recursively) apply the actions to'
+        )
+
+    ARGS = PARSER.parse_args()
+
+    if ARGS.staged_files:
+        PELOTON_DIR_bytes = bytes(PELOTON_DIR, 'utf-8')
+        TARGETS = [
+            str(os.path.abspath(os.path.join(PELOTON_DIR_bytes, f)), 'utf-8') \
+            for f in subprocess.check_output(
+                ["git", "diff", "--name-only", "HEAD", "--cached",
+                 "--diff-filter=d"
+                ]
+            ).split()]
+
+        if not TARGETS:
+            LOG.error(
+                "no staged files or not calling from a repository -- exiting"
+                )
             sys.exit("no staged files or not calling from a repository")
-    elif not args.paths:
+    elif not ARGS.paths:
         LOG.error("no files or directories given -- exiting")
         sys.exit("no files or directories given")
     else:
-        targets = args.paths
-    
-    for x in targets:
+        TARGETS = ARGS.paths
+
+    for x in TARGETS:
         if os.path.isfile(x):
-            LOG.info("Scanning file: " + x)
-            format_file(x, args.update_header, args.clang_format_code)
+            LOG.info("Scanning file: %s", x)
+            format_file(x, ARGS.update_header, ARGS.clang_format_code)
         elif os.path.isdir(x):
-            LOG.info("Scanning directory " + x)
-            format_dir(x, args.update_header, args.clang_format_code)
+            LOG.info("Scanning directory %s", x)
+            format_dir(x, ARGS.update_header, ARGS.clang_format_code)
     ## FOR
 ## IF
-        

script/formatting/formatter.py

@@ -8,18 +8,19 @@
 # To enable, symlink this file to '.git/hooks/pre-commit' like so:
 #   ln -s ../../script/git-hooks/pre-commit .git/hooks/pre-commit
 
-FORMATTER_PATH="./script/formatting/formatter.py"
+FORMATTER_COMMAND="./script/formatting/formatter.py"
 
 FILES=$(git diff --name-only HEAD --cached --diff-filter=d | grep '\.\(cpp\|h\)$')
 if [ -n "$FILES" ]; then
-  python script/validators/source_validator.py --files $FILES
+  ./script/validators/source_validator.py --files $FILES
   RESULT=$?
   if [ $RESULT -ne 0 ]; then
     echo "***************************************"
     echo "******* Peloton Pre-Commit Hook *******"
     echo "***************************************"
     echo "Use \"$FORMATTER_PATH -c -f\" to format all staged files."
     echo "Or use \"git commit --no-verify\" to temporarily bypass the pre-commit hook."
+
     echo
     echo "Be aware that changed files have to be staged again!"
     echo "***************************************"