diff --git a/src/client.py b/src/client.py --- a/src/client.py +++ b/src/client.py @@ -7,7 +7,7 @@ from datetime import datetime import config as conf import stats from util import Progress -from hashtree import HashTree +from hashtree import HashTree,hashBlock from networkers import NetworkReader,NetworkWriter @@ -36,13 +36,19 @@ class Connection: class Client: - def __init__(self,filename): + def __init__(self,filename,treeFile=""): self._incoming=None self._outcoming=None self._filename=filename + self._treeFile=treeFile print(datetime.now(), "initializing...") - self._localTree=HashTree.fromFile(self._filename) + if treeFile: + self._tree=HashTree.load(treeFile) + else: + self._tree=HashTree.fromFile(filename) + + self._newLeaves=dict() ## Asks server for node hashes to determine which are to be transferred. # @@ -50,7 +56,7 @@ class Client: # # Requests nodes in order of a batch DFS. Needs stack of size O(treeDepth*batchSize). Nodes in each tree level are accessed in order. def negotiate(self): - localTree=self._localTree + localTree=self._tree blocksToTransfer=[] nodeStack=collections.deque([0]) # root @@ -135,6 +141,9 @@ class Client: dataFile.seek(i2*HashTree.BLOCK_SIZE) dataFile.write(binData) + if self._treeFile: + self._newLeaves[i2+self._tree.leafStart]=hashBlock(binData) + log.info("block #{0}: {1}...{2}".format(i2,binData[:5],binData[-5:])) stats.logTransferredBlock() @@ -147,5 +156,11 @@ class Client: log.info("closing session...") dataFile.close() + if self._treeFile: + log.info("updating hash tree...") + for (k,v) in self._newLeaves.items(): + self._tree.updateLeaf(k, v) + self._tree.save(self._treeFile) + def setConnection(self,connection): (self._incoming,self._outcoming)=connection diff --git a/src/hashtree.py b/src/hashtree.py --- a/src/hashtree.py +++ b/src/hashtree.py @@ -5,6 +5,10 @@ from datetime import datetime from util import Progress +def hashBlock(data): + return hashlib.sha256(data).digest()[-HashTree.HASH_LEN:] + + class HashTree: HASH_LEN=16 # bytes BLOCK_SIZE=4096 # bytes @@ -28,7 +32,7 @@ class HashTree: progress=Progress(leafCount) for i in range(leafCount): data=f.read(HashTree.BLOCK_SIZE) - res.insertLeaf(hashlib.sha256(data).digest()[-HashTree.HASH_LEN:]) + res.insertLeaf(hashBlock(data)) progress.p(i) progress.done() @@ -70,7 +74,7 @@ class HashTree: ## Updates the node at index and all its ancestors. def updateNode(self,index): while index>=0: - self.store[index]=hashlib.sha256(self.store[index*2+1]+self.store[index*2+2]).digest()[-HashTree.HASH_LEN:] + self.store[index]=hashBlock(self.store[index*2+1]+self.store[index*2+2]) index=(index-1)//2 ## Fast construction of the tree over the leaves. O(n). @@ -78,14 +82,6 @@ class HashTree: print(datetime.now(), "building tree:") progress=Progress(-1, self.leafStart-1) for i in range(self.leafStart-1,-1,-1): - self.store[i]=hashlib.sha256(self.store[i*2+1]+self.store[i*2+2]).digest()[-HashTree.HASH_LEN:] + self.store[i]=hashBlock(self.store[i*2+1]+self.store[i*2+2]) progress.p(i) progress.done() - - -if __name__=="__main__": - f1=HashTree.fromFile(open("serverFile.txt",mode='rb')) - f2=HashTree.fromFile(open("clientFile.txt",mode='rb')) - - for i,(h1,h2) in enumerate(zip(f1.store,f2.store)): - print("{0:2}".format(i),h1.hex(),h2.hex(),h1==h2) diff --git a/src/morevna.py b/src/morevna.py --- a/src/morevna.py +++ b/src/morevna.py @@ -25,10 +25,12 @@ def buildTree(args): def push(args): _checkFile(args.datafile) + if args.tree: + _checkFile(args.tree) if args.host: conf.hosts.insert(0,args.host) if args.port: conf.port=args.port - c=Client(args.datafile) + c=Client(args.datafile,args.tree) with ClientConnection() as con: c.setConnection(con) blocksToTransfer=c.negotiate() @@ -38,10 +40,12 @@ def push(args): def pull(args): _checkFile(args.datafile) + if args.tree: + _checkFile(args.tree) if args.host: conf.hosts.insert(0,args.host) if args.port: conf.port=args.port - c=Client(args.datafile) + c=Client(args.datafile,args.tree) with ClientConnection() as con: c.setConnection(con) blocksToTransfer=c.negotiate() @@ -74,12 +78,14 @@ pBuild.set_defaults(func=buildTree) pUpdate=subparsers.add_parser("push") pUpdate.add_argument("-p","--port",type=int) pUpdate.add_argument("--host",default="127.0.0.1") +pUpdate.add_argument("-t","--tree",help="stored hash tree location") pUpdate.add_argument("datafile") pUpdate.set_defaults(func=push) pUpdate=subparsers.add_parser("pull") pUpdate.add_argument("-p","--port",type=int) pUpdate.add_argument("--host",default="127.0.0.1") +pUpdate.add_argument("-t","--tree",help="stored hash tree location") pUpdate.add_argument("datafile") pUpdate.set_defaults(func=pull) diff --git a/src/server.py b/src/server.py --- a/src/server.py +++ b/src/server.py @@ -1,10 +1,9 @@ -import hashlib -import socket +import socket import ssl import multiprocessing import logging as log -from hashtree import HashTree +from hashtree import HashTree,hashBlock from networkers import NetworkReader,NetworkWriter import config as conf @@ -140,7 +139,7 @@ class Server: self._dataFile.write(binData) self._lastIndex=i if self._treeFile: - self._newLeaves[i+self._tree.leafStart]=hashlib.sha256(binData).digest()[HashTree.HASH_LEN:] + self._newLeaves[i+self._tree.leafStart]=hashBlock(binData) return ({"command": "ack", "index": i},)