diff --git a/game_world.jpg b/game_world.jpg new file mode 100644 index 0000000..12dff17 Binary files /dev/null and b/game_world.jpg differ diff --git a/proto/chunk.proto b/proto/chunk.proto index 05747d6..e83ca79 100644 --- a/proto/chunk.proto +++ b/proto/chunk.proto @@ -7,7 +7,8 @@ service FileServer { } message Chunk { - bytes buffer = 1; + string hash = 1; + bytes buffer = 2; } message Request { @@ -15,5 +16,6 @@ message Request { } message Reply { - int32 length = 1; + string msg = 1; + int32 length = 2; } diff --git a/proto/codegen.sh b/proto/codegen.sh old mode 100644 new mode 100755 index df6bdef..6251fc4 --- a/proto/codegen.sh +++ b/proto/codegen.sh @@ -1 +1 @@ -python -m grpc_tools.protoc -I. --python_out=../src --grpc_python_out=../src ./chunk.proto +python3 -m grpc_tools.protoc -I. --python_out=../src --pyi_out=../src --grpc_python_out=../src ./chunk.proto diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..e3f11ba --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +grpcio==1.51.1 +grpcio-tools==1.51.1 +pkg_resources==0.0.0 +protobuf==4.21.12 diff --git a/src/chunk_pb2.py b/src/chunk_pb2.py index 2ce76e3..42bc472 100644 --- a/src/chunk_pb2.py +++ b/src/chunk_pb2.py @@ -1,13 +1,11 @@ +# -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: chunk.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +"""Generated protocol buffer code.""" +from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection +from google.protobuf import descriptor_pool as _descriptor_pool from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() @@ -15,166 +13,19 @@ -DESCRIPTOR = _descriptor.FileDescriptor( - name='chunk.proto', - package='', - syntax='proto3', - serialized_pb=_b('\n\x0b\x63hunk.proto\"\x17\n\x05\x43hunk\x12\x0e\n\x06\x62uffer\x18\x01 \x01(\x0c\"\x17\n\x07Request\x12\x0c\n\x04name\x18\x01 \x01(\t\"\x17\n\x05Reply\x12\x0e\n\x06length\x18\x01 \x01(\x05\x32L\n\nFileServer\x12\x1c\n\x06upload\x12\x06.Chunk\x1a\x06.Reply\"\x00(\x01\x12 \n\x08\x64ownload\x12\x08.Request\x1a\x06.Chunk\"\x00\x30\x01\x62\x06proto3') -) - - - - -_CHUNK = _descriptor.Descriptor( - name='Chunk', - full_name='Chunk', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='buffer', full_name='Chunk.buffer', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=15, - serialized_end=38, -) - - -_REQUEST = _descriptor.Descriptor( - name='Request', - full_name='Request', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='Request.name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=40, - serialized_end=63, -) - - -_REPLY = _descriptor.Descriptor( - name='Reply', - full_name='Reply', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='length', full_name='Reply.length', index=0, - number=1, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=65, - serialized_end=88, -) - -DESCRIPTOR.message_types_by_name['Chunk'] = _CHUNK -DESCRIPTOR.message_types_by_name['Request'] = _REQUEST -DESCRIPTOR.message_types_by_name['Reply'] = _REPLY -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -Chunk = _reflection.GeneratedProtocolMessageType('Chunk', (_message.Message,), dict( - DESCRIPTOR = _CHUNK, - __module__ = 'chunk_pb2' - # @@protoc_insertion_point(class_scope:Chunk) - )) -_sym_db.RegisterMessage(Chunk) - -Request = _reflection.GeneratedProtocolMessageType('Request', (_message.Message,), dict( - DESCRIPTOR = _REQUEST, - __module__ = 'chunk_pb2' - # @@protoc_insertion_point(class_scope:Request) - )) -_sym_db.RegisterMessage(Request) - -Reply = _reflection.GeneratedProtocolMessageType('Reply', (_message.Message,), dict( - DESCRIPTOR = _REPLY, - __module__ = 'chunk_pb2' - # @@protoc_insertion_point(class_scope:Reply) - )) -_sym_db.RegisterMessage(Reply) - - - -_FILESERVER = _descriptor.ServiceDescriptor( - name='FileServer', - full_name='FileServer', - file=DESCRIPTOR, - index=0, - options=None, - serialized_start=90, - serialized_end=166, - methods=[ - _descriptor.MethodDescriptor( - name='upload', - full_name='FileServer.upload', - index=0, - containing_service=None, - input_type=_CHUNK, - output_type=_REPLY, - options=None, - ), - _descriptor.MethodDescriptor( - name='download', - full_name='FileServer.download', - index=1, - containing_service=None, - input_type=_REQUEST, - output_type=_CHUNK, - options=None, - ), -]) -_sym_db.RegisterServiceDescriptor(_FILESERVER) +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0b\x63hunk.proto\"%\n\x05\x43hunk\x12\x0c\n\x04hash\x18\x01 \x01(\t\x12\x0e\n\x06\x62uffer\x18\x02 \x01(\x0c\"\x17\n\x07Request\x12\x0c\n\x04name\x18\x01 \x01(\t\"$\n\x05Reply\x12\x0b\n\x03msg\x18\x01 \x01(\t\x12\x0e\n\x06length\x18\x02 \x01(\x05\x32L\n\nFileServer\x12\x1c\n\x06upload\x12\x06.Chunk\x1a\x06.Reply\"\x00(\x01\x12 \n\x08\x64ownload\x12\x08.Request\x1a\x06.Chunk\"\x00\x30\x01\x62\x06proto3') -DESCRIPTOR.services_by_name['FileServer'] = _FILESERVER +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'chunk_pb2', globals()) +if _descriptor._USE_C_DESCRIPTORS == False: + DESCRIPTOR._options = None + _CHUNK._serialized_start=15 + _CHUNK._serialized_end=52 + _REQUEST._serialized_start=54 + _REQUEST._serialized_end=77 + _REPLY._serialized_start=79 + _REPLY._serialized_end=115 + _FILESERVER._serialized_start=117 + _FILESERVER._serialized_end=193 # @@protoc_insertion_point(module_scope) diff --git a/src/chunk_pb2.pyi b/src/chunk_pb2.pyi new file mode 100644 index 0000000..f4cba9a --- /dev/null +++ b/src/chunk_pb2.pyi @@ -0,0 +1,27 @@ +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from typing import ClassVar as _ClassVar, Optional as _Optional + +DESCRIPTOR: _descriptor.FileDescriptor + +class Chunk(_message.Message): + __slots__ = ["buffer", "hash"] + BUFFER_FIELD_NUMBER: _ClassVar[int] + HASH_FIELD_NUMBER: _ClassVar[int] + buffer: bytes + hash: str + def __init__(self, hash: _Optional[str] = ..., buffer: _Optional[bytes] = ...) -> None: ... + +class Reply(_message.Message): + __slots__ = ["length", "msg"] + LENGTH_FIELD_NUMBER: _ClassVar[int] + MSG_FIELD_NUMBER: _ClassVar[int] + length: int + msg: str + def __init__(self, msg: _Optional[str] = ..., length: _Optional[int] = ...) -> None: ... + +class Request(_message.Message): + __slots__ = ["name"] + NAME_FIELD_NUMBER: _ClassVar[int] + name: str + def __init__(self, name: _Optional[str] = ...) -> None: ... diff --git a/src/chunk_pb2_grpc.py b/src/chunk_pb2_grpc.py index deb223e..d14c17b 100644 --- a/src/chunk_pb2_grpc.py +++ b/src/chunk_pb2_grpc.py @@ -1,63 +1,99 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" import grpc import chunk_pb2 as chunk__pb2 class FileServerStub(object): - # missing associated documentation comment in .proto file - pass - - def __init__(self, channel): - """Constructor. - - Args: - channel: A grpc.Channel. - """ - self.upload = channel.stream_unary( - '/FileServer/upload', - request_serializer=chunk__pb2.Chunk.SerializeToString, - response_deserializer=chunk__pb2.Reply.FromString, - ) - self.download = channel.unary_stream( - '/FileServer/download', - request_serializer=chunk__pb2.Request.SerializeToString, - response_deserializer=chunk__pb2.Chunk.FromString, - ) + """Missing associated documentation comment in .proto file.""" + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.upload = channel.stream_unary( + '/FileServer/upload', + request_serializer=chunk__pb2.Chunk.SerializeToString, + response_deserializer=chunk__pb2.Reply.FromString, + ) + self.download = channel.unary_stream( + '/FileServer/download', + request_serializer=chunk__pb2.Request.SerializeToString, + response_deserializer=chunk__pb2.Chunk.FromString, + ) class FileServerServicer(object): - # missing associated documentation comment in .proto file - pass + """Missing associated documentation comment in .proto file.""" - def upload(self, request_iterator, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') + def upload(self, request_iterator, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') - def download(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') + def download(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') def add_FileServerServicer_to_server(servicer, server): - rpc_method_handlers = { - 'upload': grpc.stream_unary_rpc_method_handler( - servicer.upload, - request_deserializer=chunk__pb2.Chunk.FromString, - response_serializer=chunk__pb2.Reply.SerializeToString, - ), - 'download': grpc.unary_stream_rpc_method_handler( - servicer.download, - request_deserializer=chunk__pb2.Request.FromString, - response_serializer=chunk__pb2.Chunk.SerializeToString, - ), - } - generic_handler = grpc.method_handlers_generic_handler( - 'FileServer', rpc_method_handlers) - server.add_generic_rpc_handlers((generic_handler,)) + rpc_method_handlers = { + 'upload': grpc.stream_unary_rpc_method_handler( + servicer.upload, + request_deserializer=chunk__pb2.Chunk.FromString, + response_serializer=chunk__pb2.Reply.SerializeToString, + ), + 'download': grpc.unary_stream_rpc_method_handler( + servicer.download, + request_deserializer=chunk__pb2.Request.FromString, + response_serializer=chunk__pb2.Chunk.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'FileServer', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + + + # This class is part of an EXPERIMENTAL API. +class FileServer(object): + """Missing associated documentation comment in .proto file.""" + + @staticmethod + def upload(request_iterator, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.stream_unary(request_iterator, target, '/FileServer/upload', + chunk__pb2.Chunk.SerializeToString, + chunk__pb2.Reply.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def download(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream(request, target, '/FileServer/download', + chunk__pb2.Request.SerializeToString, + chunk__pb2.Chunk.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/src/demo_client.py b/src/demo_client.py index 2730c8b..e88ac3d 100644 --- a/src/demo_client.py +++ b/src/demo_client.py @@ -7,7 +7,7 @@ client = lib.FileClient('localhost:8888') # demo for file uploading - in_file_name = '/tmp/large_file_in' + in_file_name = 'game_world.jpg' client.upload(in_file_name) # demo for file downloading: @@ -15,5 +15,6 @@ if os.path.exists(out_file_name): os.remove(out_file_name) client.download('whatever_name', out_file_name) + print() os.system(f'sha1sum {in_file_name}') os.system(f'sha1sum {out_file_name}') diff --git a/src/lib.py b/src/lib.py index c281e4a..38fb6dc 100644 --- a/src/lib.py +++ b/src/lib.py @@ -3,25 +3,47 @@ import grpc import time +import hashlib import chunk_pb2, chunk_pb2_grpc CHUNK_SIZE = 1024 * 1024 # 1MB +def get_sha256_file(filepath): + BLOCK_SIZE = 1024 + m = hashlib.sha256() + if os.path.exists(filepath): + with open(filepath,"rb") as myfile: + while True: + data = myfile.read(BLOCK_SIZE) + if not data: + return m + m.update(data) + + def get_file_chunks(filename): with open(filename, 'rb') as f: while True: - piece = f.read(CHUNK_SIZE); + piece = f.read(CHUNK_SIZE) if len(piece) == 0: return - yield chunk_pb2.Chunk(buffer=piece) + yield chunk_pb2.Chunk(hash=get_sha256_file(filename).hexdigest(),buffer=piece) def save_chunks_to_file(chunks, filename): + # write chunks to file with open(filename, 'wb') as f: for chunk in chunks: + if chunk.hash != 0: + hash_data = chunk.hash f.write(chunk.buffer) + # integrity check - compute and compare hashes + if hash_data == get_sha256_file(filename).hexdigest(): + return True + else: + return False + class FileClient: @@ -30,9 +52,18 @@ def __init__(self, address): self.stub = chunk_pb2_grpc.FileServerStub(channel) def upload(self, in_file_name): + # check if file exists and has read permissions + if os.path.isfile(in_file_name) and os.access(in_file_name, os.R_OK): + print("[CLIENT] File exists and is readable") + else: + print("[CLIENT] Either the file is missing or not readable") + exit() + chunks_generator = get_file_chunks(in_file_name) response = self.stub.upload(chunks_generator) assert response.length == os.path.getsize(in_file_name) + print(f"[SERVER] {response.msg}") + print(f"[SERVER] {response.length} bytes recieved") def download(self, target_name, out_file_name): response = self.stub.download(chunk_pb2.Request(name=target_name)) @@ -47,8 +78,12 @@ def __init__(self): self.tmp_file_name = '/tmp/server_tmp' def upload(self, request_iterator, context): - save_chunks_to_file(request_iterator, self.tmp_file_name) - return chunk_pb2.Reply(length=os.path.getsize(self.tmp_file_name)) + if save_chunks_to_file(request_iterator, self.tmp_file_name): + return chunk_pb2.Reply(msg="Data received and verified.",length=os.path.getsize(self.tmp_file_name)) + else: + return chunk_pb2.Reply(msg="Data received but verification failed!",length=os.path.getsize(self.tmp_file_name)) + + def download(self, request, context): if request.name: