Fix merge conflict in restapi.py

Author: cpacia

api/restapi.py

@@ -266,9 +266,9 @@ def get_node(node):
     @authenticated
     def get_followers(self, request):
         def parse_followers(followers):
-            if followers is not None:
+            if followers[0] is not None:
                 response = {"followers": []}
-                for f in followers.followers:
+                for f in followers[0].followers:
                     follower_json = {
                         "guid": f.guid.encode("hex"),
                         "handle": f.metadata.handle,
@@ -278,28 +278,33 @@ def parse_followers(followers):
                         "nsfw": f.metadata.nsfw
                     }
                     response["followers"].append(follower_json)
+                if followers[1] is not None:
+                    response["count"] = followers[1]
                 request.setHeader('content-type', "application/json")
                 request.write(json.dumps(sanitize_html(response), indent=4))
                 request.finish()
             else:
                 request.write(json.dumps({}))
                 request.finish()
+        start = 0
+        if "start" in request.args:
+            start = int(request.args["start"][0])
         if "guid" in request.args:
             def get_node(node):
                 if node is not None:
-                    self.mserver.get_followers(node).addCallback(parse_followers)
+                    self.mserver.get_followers(node, start).addCallback(parse_followers)
                 else:
                     request.write(json.dumps({}))
                     request.finish()
             self.kserver.resolve(unhexlify(request.args["guid"][0])).addCallback(get_node)
         else:
-            ser = self.db.follow.get_followers()
-            if ser is not None:
+            ser = self.db.follow.get_followers(start)
+            if ser[0] is not None:
                 f = objects.Followers()
-                f.ParseFromString(ser)
-                parse_followers(f)
+                f.ParseFromString(ser[0])
+                parse_followers((f, ser[1]))
             else:
-                parse_followers(None)
+                parse_followers((None, 0))
         return server.NOT_DONE_YET
 
     @GET('^/api/v1/get_following')

config.py

@@ -12,7 +12,7 @@
 from ConfigParser import ConfigParser
 from urlparse import urlparse
 
-PROTOCOL_VERSION = 1
+PROTOCOL_VERSION = 2
 CONFIG_FILE = join(os.getcwd(), 'ob.cfg')
 
 # FIXME probably a better way to do this. This curretly checks two levels deep.

db/datastore.py

@@ -10,7 +10,7 @@
 from protos import objects
 from protos.objects import Listings, Followers, Following
 from os.path import join
-from db.migrations import migration1, migration2
+from db.migrations import migration1, migration2, migration3
 
 
 class Database(object):
@@ -116,7 +116,7 @@ def _create_database(database_path):
         conn = lite.connect(database_path)
         cursor = conn.cursor()
 
-        cursor.execute('''PRAGMA user_version = 2''')
+        cursor.execute('''PRAGMA user_version = 3''')
         cursor.execute('''CREATE TABLE hashmap(hash TEXT PRIMARY KEY, filepath TEXT)''')
 
         cursor.execute('''CREATE TABLE profile(id INTEGER PRIMARY KEY, serializedUserInfo BLOB, tempHandle TEXT)''')
@@ -125,7 +125,8 @@ def _create_database(database_path):
 
         cursor.execute('''CREATE TABLE keys(type TEXT PRIMARY KEY, privkey BLOB, pubkey BLOB)''')
 
-        cursor.execute('''CREATE TABLE followers(id INTEGER PRIMARY KEY, serializedFollowers BLOB)''')
+        cursor.execute('''CREATE TABLE followers(guid TEXT UNIQUE, serializedFollower TEXT)''')
+        cursor.execute('''CREATE INDEX index_followers ON followers(serializedFollower);''')
 
         cursor.execute('''CREATE TABLE following(id INTEGER PRIMARY KEY, serializedFollowing BLOB)''')
 
@@ -182,8 +183,12 @@ def _run_migrations(self):
         if version == 0:
             migration1.migrate(self.PATH)
             migration2.migrate(self.PATH)
+            migration3.migrate(self.PATH)
         elif version == 1:
             migration2.migrate(self.PATH)
+            migration3.migrate(self.PATH)
+        elif version == 2:
+            migration3.migrate(self.PATH)
 
 
 class HashMap(object):
@@ -466,47 +471,41 @@ def is_following(self, guid):
 
     def set_follower(self, proto):
         conn = Database.connect_database(self.PATH)
+        p = Followers.Follower()
+        p.ParseFromString(proto)
         with conn:
             cursor = conn.cursor()
-            f = Followers()
-            ser = self.get_followers()
-            if ser is not None:
-                f.ParseFromString(ser)
-                for follower in f.followers:
-                    if follower.guid == proto.guid:
-                        f.followers.remove(follower)
-            f.followers.extend([proto])
-            cursor.execute('''INSERT OR REPLACE INTO followers(id, serializedFollowers) VALUES (?,?)''',
-                           (1, f.SerializeToString()))
+            cursor.execute('''INSERT OR REPLACE INTO followers(guid, serializedFollower) VALUES (?,?)''',
+                           (p.guid.encode("hex"), proto.encode("hex")))
             conn.commit()
         conn.close()
 
     def delete_follower(self, guid):
         conn = Database.connect_database(self.PATH)
         with conn:
             cursor = conn.cursor()
-            f = Followers()
-            ser = self.get_followers()
-            if ser is not None:
-                f.ParseFromString(ser)
-                for follower in f.followers:
-                    if follower.guid == guid:
-                        f.followers.remove(follower)
-            cursor.execute('''INSERT OR REPLACE INTO followers(id, serializedFollowers) VALUES (?,?)''',
-                           (1, f.SerializeToString()))
+            cursor.execute('''DELETE FROM followers WHERE guid=?''', (guid.encode("hex"), ))
             conn.commit()
         conn.close()
 
-    def get_followers(self):
+    def get_followers(self, start=0):
         conn = Database.connect_database(self.PATH)
         cursor = conn.cursor()
-        cursor.execute('''SELECT serializedFollowers FROM followers WHERE id=1''')
-        proto = cursor.fetchone()
-        conn.close()
-        if not proto:
-            return None
-        else:
-            return proto[0]
+
+        cursor.execute('''SELECT Count(*) FROM followers''')
+        count = cursor.fetchone()[0]
+
+        f = Followers()
+        if count > 0:
+            smt = '''select serializedFollower from followers order by rowid desc limit 30 offset ''' + str(start)
+            cursor.execute(smt)
+            serialized_followers = cursor.fetchall()
+            conn.close()
+            for proto in serialized_followers:
+                p = Followers.Follower()
+                p.ParseFromString(proto[0].decode("hex"))
+                f.followers.extend([p])
+        return (f.SerializeToString(), count)
 
 
 class MessageStore(object):
@@ -591,7 +590,7 @@ def get_conversations(self):
             handle = ""
             if val[0] is not None:
                 try:
-                    with open(join(DATA_FOLDER, 'cache', g[0]), "r") as filename:
+                    with open(join(DATA_FOLDER, 'cache', g[0] + ".profile"), "r") as filename:
                         profile = filename.read()
                     p = objects.Profile()
                     p.ParseFromString(profile)

db/migrations/migration2.py

@@ -21,4 +21,5 @@ def migrate(database_path):
     # update version
     cursor.execute('''PRAGMA user_version = 2''')
     conn.commit()
+
     conn.close()

db/migrations/migration3.py

@@ -0,0 +1,33 @@
+import sqlite3
+from protos import objects
+
+
+def migrate(database_path):
+    print "migrating to db version 3"
+    conn = sqlite3.connect(database_path)
+    conn.text_factory = str
+    cursor = conn.cursor()
+    # read followers from db
+    cursor.execute('''SELECT serializedFollowers FROM followers WHERE id=1''')
+    followers = cursor.fetchone()
+
+    # delete follower table
+    cursor.execute('''DROP TABLE followers''')
+
+    # create new table
+    cursor.execute('''CREATE TABLE followers(guid TEXT UNIQUE, serializedFollower TEXT)''')
+    cursor.execute('''CREATE INDEX index_followers ON followers(serializedFollower);''')
+
+    # write followers back into db
+
+    if followers is not None:
+        f = objects.Followers()
+        f.ParseFromString(followers[0])
+        for follower in f.followers:
+            cursor.execute('''INSERT INTO followers(guid, serializedFollower) VALUES (?,?)''',
+                           (follower.guid.encode("hex"), follower.SerializeToString().encode("hex"),))
+
+    # update version
+    cursor.execute('''PRAGMA user_version = 3''')
+    conn.commit()
+    conn.close()

db/tests/test_datastore.py

@@ -163,13 +163,13 @@ def test_follow_unfollow(self):
         self.assertFalse(self.fd.is_following(self.u.guid))
 
     def test_deleteFollower(self):
-        self.fd.set_follower(self.f)
-        self.fd.set_follower(self.f)
+        self.fd.set_follower(self.f.SerializeToString())
+        self.fd.set_follower(self.f.SerializeToString())
         f = self.fd.get_followers()
         self.assertIsNotNone(f)
         self.fd.delete_follower(self.f.guid)
         f = self.fd.get_followers()
-        self.assertEqual(f, '')
+        self.assertEqual(f[0], '')
 
     def test_MassageStore(self):
         msgs = self.ms.get_messages(self.u.guid, 'CHAT')

market/network.py

@@ -195,7 +195,7 @@ def get_result(result):
                     self.get_image(node_to_ask, p.avatar_hash)
                 if not os.path.isfile(os.path.join(DATA_FOLDER, 'cache', p.header_hash.encode("hex"))):
                     self.get_image(node_to_ask, p.header_hash)
-                self.cache(result[1][0], node_to_ask.id.encode("hex"))
+                self.cache(result[1][0], node_to_ask.id.encode("hex") + ".profile")
                 return p
             except Exception:
                 return None
@@ -373,7 +373,7 @@ def save_to_db(result):
         self.log.info("sending unfollow request to %s" % node_to_unfollow)
         return d.addCallback(save_to_db)
 
-    def get_followers(self, node_to_ask):
+    def get_followers(self, node_to_ask, start=0):
         """
         Query the given node for a list if its followers. The response will be a
         `Followers` protobuf object. We will verify the signature for each follower
@@ -388,8 +388,11 @@ def get_response(response):
                 verify_key.verify(response[1][0], response[1][1])
                 f.ParseFromString(response[1][0])
             except Exception:
-                return None
+                return (None, None)
             # Verify the signature and guid of each follower.
+            count = None
+            if len(response[1]) > 2:
+                count = response[1][2]
             for follower in f.followers:
                 try:
                     v_key = nacl.signing.VerifyKey(follower.pubkey)
@@ -404,9 +407,14 @@ def get_response(response):
                         raise Exception('Invalid follower')
                 except Exception:
                     f.followers.remove(follower)
-            return f
+            return (f, count)
 
-        d = self.protocol.callGetFollowers(node_to_ask)
+        peer = (node_to_ask.ip, node_to_ask.port)
+        if peer in self.protocol.multiplexer and \
+                        self.protocol.multiplexer[peer].handler.remote_node_version > 1:
+            d = self.protocol.callGetFollowers(node_to_ask, start=start)
+        else:
+            d = self.protocol.callGetFollowers(node_to_ask)
         self.log.info("fetching followers from %s" % node_to_ask)
         return d.addCallback(get_response)
 
@@ -1241,3 +1249,15 @@ def cache(file_to_save, filename):
         """
         with open(os.path.join(DATA_FOLDER, "cache", filename), 'wb') as outfile:
             outfile.write(file_to_save)
+
+    @staticmethod
+    def load_from_cache(filename):
+        """
+        Loads a file from cache
+        """
+        filepath = DATA_FOLDER + "cache/" + filename
+        if not os.path.exists(filepath):
+            return None
+        with open(filepath, "r") as filename:
+            f = filename.read()
+        return f

market/protocol.py

@@ -175,14 +175,14 @@ def rpc_unfollow(self, sender, signature):
             self.log.warning("failed to validate signature on unfollow request")
             return ["False"]
 
-    def rpc_get_followers(self, sender):
+    def rpc_get_followers(self, sender, start=None):
         self.log.info("serving followers list to %s" % sender)
         self.router.addContact(sender)
-        ser = self.db.follow.get_followers()
-        if ser is None:
-            return None
+        if start is not None:
+            ser = self.db.follow.get_followers(int(start))
         else:
-            return [ser, self.signing_key.sign(ser)[:64]]
+            ser = self.db.follow.get_followers()
+        return [ser[0], self.signing_key.sign(ser[0])[:64], ser[1]]
 
     def rpc_get_following(self, sender):
         self.log.info("serving following list to %s" % sender)
@@ -398,8 +398,11 @@ def callUnfollow(self, nodeToAsk, signature):
         d = self.unfollow(nodeToAsk, signature)
         return d.addCallback(self.handleCallResponse, nodeToAsk)
 
-    def callGetFollowers(self, nodeToAsk):
-        d = self.get_followers(nodeToAsk)
+    def callGetFollowers(self, nodeToAsk, start=None):
+        if start is None:
+            d = self.get_followers(nodeToAsk)
+        else:
+            d = self.get_followers(nodeToAsk, start)
         return d.addCallback(self.handleCallResponse, nodeToAsk)
 
     def callGetFollowing(self, nodeToAsk):

net/rpcudp.py

@@ -50,12 +50,6 @@ def receive_message(self, message, sender, connection, ban_score):
             connection.shutdown()
             return False
 
-        if message.protoVer < PROTOCOL_VERSION:
-            self.log.warning("received message from %s with incompatible protocol version." %
-                             str(connection.dest_addr))
-            connection.shutdown()
-            return False
-
         self.multiplexer.vendors[sender.id] = sender
 
         msgID = message.messageID

net/wireprotocol.py

@@ -68,6 +68,7 @@ def __init__(self, processors, nat_type, relay_node, ban_score, *args, **kwargs)
             self.is_new_node = True
             self.on_connection_made()
             self.time_last_message = 0
+            self.remote_node_version = 1
             self.ping_interval = 30 if nat_type != FULL_CONE else 300
 
         def on_connection_made(self):
@@ -94,6 +95,7 @@ def receive_message(self, datagram):
                                  (m.sender.relayAddress.ip, m.sender.relayAddress.port),
                                  m.sender.natType,
                                  m.sender.vendor)
+                self.remote_node_version = m.protoVer
                 if self.time_last_message == 0:
                     h = nacl.hash.sha512(m.sender.publicKey)
                     pow_hash = h[40:]