Merge pull request #4 from KrishnanSG/networking

Interact, a basic working version. (v1.0)

Author: KrishnanSG

.gitignore

@@ -1,3 +1,6 @@
+# output BloomFilter file
+*.bin
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
@@ -86,6 +89,7 @@ celerybeat-schedule
 .venv
 env/
 venv/
+*venv/
 ENV/
 env.bak/
 venv.bak/

BloomFilter.py

@@ -16,23 +16,39 @@ def __init__(self, n):
         self.size = math.ceil(-n*math.log(self.p)/(math.log(2)**2))
         self.k = math.ceil(self.size/n*math.log(2))
         self.bit_array = [0] * self.size
+        self.validate_array = []
 
     # Func to insert values into BF
-    def insert(self, value):
+    def insert(self, value, freq=1):
+        line_hash = str(mmh3.hash(value,freq))
         for i in range(self.k):
-            index = mmh3.hash(value, i) % self.size
+            index = mmh3.hash(line_hash,i) % self.size
             self.bit_array[index] = 1
 
     # To check if the value is present in BF or not
-    def validate(self, value):
+    def validate(self, value, freq=1):
+        line_hash = str(mmh3.hash(value,freq))
         for i in range(self.k):
-            check_at_index = mmh3.hash(value, i) % self.size
-            if self.bit_array[check_at_index] == 1:
+            check_at_index = mmh3.hash(line_hash,i) % self.size
+            if self.validate_array[check_at_index] == 1:
                 continue
             else:
                 return False
         return True
 
+    def readBloomFilterFromFile(self,filename):
+        f = open(filename, "rb")
+        self.validate_array = list(f.read())
+        for i in range(0, len(self.validate_array)):
+            self.validate_array[i] -= 48
+        print(self.validate_array)
+        f.close()
+    
+    def readBloomFilterFromBytes(self,bf_as_bytes):
+        self.validate_array = list(bf_as_bytes)
+        for i in range(0, len(self.validate_array)):
+            self.validate_array[i] -= 48
+
     # Returns the bit array
     def getBloomFilter(self):
         return self.bit_array
@@ -41,6 +57,12 @@ def getBloomFilter(self):
     def getSize(self):
         return self.size
 
+    def getNFromSize(self,size):
+        return(math.floor(size*-1*(math.log(2)**2)/math.log(self.p)))
+
     # Returns the # of Hash Functions ie. h1(k), h2(k) ...
     def getNumberOfHashFunctions(self):
         return self.k
+
+    def getAsBytes(self):
+        return str.encode(''.join([str(i) for i in self.bit_array]))

P2P/README.md

@@ -0,0 +1,22 @@
+## Messaging format
+
+Each message will have this format
+```
+1. First byte specifying the message type - int.
+    - `2` - Bloom filter request message from client to server - server should reply with type `3` followed by the bloomfilter.
+    - `3` - Denotes that a bloom filter is incoming
+    - `4` - Denotes that the actual changes are coming through in that request
+
+```
+
+Maybe pad all the messages with some delimiter bytes to seperate the different requests
+
+Once the BF is exchanged
+1. Call getMissingContent for both users
+2. Send the missing contents of user 1 to user 2
+3. Call merge function on user2 side
+4. Send entire file sha256 and check on both ends
+    if equal
+        done
+    else
+        send the entire file

P2P/Server.py

@@ -0,0 +1,62 @@
+import socket
+import sys
+import time
+
+from . import utils
+
+MAX_RECEIVABLE_CONTENT_SIZE = 500000
+class NetworkManager:
+
+    # Variable to tell if the current user is host or client
+    status = ""
+    queue = []
+
+    def __init__(self, request_handler):
+        self.handler = request_handler
+
+    def check_if_incoming_data(self):
+        try:
+            data = self.socket.recv(MAX_RECEIVABLE_CONTENT_SIZE)
+            req = utils.parse_received_data(data)
+            self.handler.handle_request(req)
+            return tuple([00, ''])
+        except BlockingIOError as e:
+            # print(e)
+            # No data to read
+            return tuple([00, ''])
+        except (ConnectionResetError, ValueError) as e:
+            ## ValueError is raised from parse_received_data
+            print("\n\nUh-oh. looks like peer has closed the connection :(")
+            exit()
+
+    def send_request(self, request):
+            self.socket.send(request.get_type_byte() + request.get_message_bytes())
+
+    def create_host(self):
+        self.status="host"
+        server = socket.socket()
+        host = socket.gethostname()
+
+        server.bind((host, 5050))
+        self.socket = server
+        print("\nHi ",host," we have hosted the server at")
+        print("IP:",server.getsockname()[0])
+        print("Port:",server.getsockname()[1])
+        print("Share IP and Port with your friend to start sync.\n")
+        server.listen()
+
+        c, client_addr = server.accept()
+        self.socket = c
+        print("Incoming request from", client_addr, "...")
+        print("Connected to ", client_addr, "!")
+        c.setblocking(False)            
+
+
+    def create_client(self,ip,port):
+        self.status="client"
+        client = socket.socket()
+        self.socket = client
+        host = socket.gethostname()
+        client.connect((ip,port))
+        client.setblocking(False)
+        print("\nHi ",host," you are succesfully connected to")

P2P/__init__.py

@@ -0,0 +1,54 @@
+
+
+
+class Request:
+    
+    REQUEST_TYPE_BLOOMFILTER  = 2
+    REQUEST_TYPE_REPLY_SLAVE_BLOOMFILTER = 3
+    REQUEST_SEND_ACTUAL_LINES = 4
+    REQUEST_SEND_ENTIRE_FILE_HASH = 5
+    REQUEST_SEND_ENTIRE_FILE = 6
+
+    def __init__(self, request_type, message):
+        self.type = request_type
+        self.message = message
+        if isinstance(message, str):
+            self.byte_message = bytes(message, 'utf-8')
+        elif isinstance(message, bytes) :
+            self.byte_message = message
+        else:
+            self.byte_message = bytes(message)
+
+    def get_type_byte(self):
+        return bytes([self.type])
+
+    def get_type(self):
+        return self.type
+
+    def get_message_size(self):
+        return len(self.actual_message())
+
+    def actual_message(self):
+        return self.byte_message.decode('utf-8')
+
+    def get_message_bytes(self):
+        return self.byte_message
+
+    def __str__(self):
+        return "<Request type: " + str(self.type) +  ", message: " + self.actual_message() + "...>"
+
+
+
+def parse_received_data(data):
+    type_specifying_byte = data[0:1]
+    bloom_filter = data[1:]
+    try:
+        str_message = bloom_filter.decode('utf-8')
+    except UnicodeDecodeError:
+        # Occurs sometimes when the contents received are not in utf-8
+        # This can happen for example, when transmitting the hash value.
+        str_message = bloom_filter
+    print(bytes.hex(type_specifying_byte))
+    type_int = int(bytes.hex(type_specifying_byte), 16)
+    req = Request(type_int, str_message)
+    return req

P2P/received_file

@@ -0,0 +1,52 @@
+import mmh3
+def syncFile(filename,my_missing_content,received_missing_content):
+    
+    file_content=[]
+    contents_to_modify=[]
+
+    # Read file
+    with open(filename,'r') as f:
+        file_content=f.readlines()
+
+    for i in received_missing_content:
+        if i in my_missing_content:
+            # Present so update contents
+            file_content[i-1]=received_missing_content[i]
+            del(my_missing_content[i])
+        else:
+            # Not Present so insert no line
+            # lines_to_insert.append(i)
+            contents_to_modify.append([i,'i',received_missing_content[i]])
+
+    for i in my_missing_content.keys():
+        contents_to_modify.append([i,'d'])
+    contents_to_modify.sort()
+
+    def update_lines(flag):
+        for i in range(len(contents_to_modify)):
+            if contents_to_modify[i][1]=='d':
+                contents_to_modify[i][0]+=flag
+        
+        
+    while contents_to_modify:
+        i=contents_to_modify.pop(0)
+        # Insert
+        if i[1]=='i':
+            file_content.insert(i[0]-1,i[2])
+            update_lines(1)
+        # Delete
+        else:
+            file_content.pop(i[0]-1)
+            update_lines(-1)
+
+    # Write into file
+    with open(filename,'w') as f:
+        f.writelines(file_content)
+
+def computeHash(filename):
+    # returns 4 byte hash value of the file
+    file_reader = open(filename,"rb")
+    file_contents = file_reader.read()
+    n=abs(mmh3.hash(file_contents)) % 2**32
+    return (n.to_bytes((n.bit_length()+7)//8,'big'))
+