Changeset - 8bb6a904d50b
Parent rev.
Child rev.
[Not reviewed]
default
0 4 0
Laman - 8 years ago 2017-10-19 13:26:42

client too can now use saved hash tree
4 files changed with 37 insertions and 21 deletions:
src/client.py
19
4
src/hashtree.py
7
11
src/morevna.py
8
2
src/server.py
3
4
0 comments (0 inline, 0 general)
src/client.py
➞
Show inline comments
 
@@ -7,7 +7,7 @@ from datetime import datetime
 
import config as conf
 
import stats
 
from util import Progress
 
from hashtree import HashTree
 
from hashtree import HashTree,hashBlock
 
from networkers import NetworkReader,NetworkWriter
 

			




	
	
	
		
 

			




	
	
		
 
@@ -36,13 +36,19 @@ class Connection:

			




	
	
	
		
 

			




	
	
	
		
 

			




	
	
	
		
 
class Client:

			




	
	
	
		
 
	def __init__(self,filename):

			




	
	
	
		
 
	def __init__(self,filename,treeFile=""):

			




	
	
	
		
 
		self._incoming=None

			




	
	
	
		
 
		self._outcoming=None

			




	
	
	
		
 
		self._filename=filename

			




	
	
	
		
 
		self._treeFile=treeFile

			




	
	
	
		
 

			




	
	
	
		
 
		print(datetime.now(), "initializing...")

			




	
	
	
		
 
		self._localTree=HashTree.fromFile(self._filename)

			




	
	
	
		
 
		if treeFile:

			




	
	
	
		
 
			self._tree=HashTree.load(treeFile)

			




	
	
	
		
 
		else:

			




	
	
	
		
 
			self._tree=HashTree.fromFile(filename)

			




	
	
	
		
 

			




	
	
	
		
 
		self._newLeaves=dict()

			




	
	
	
		
 

			




	
	
	
		
 
	## Asks server for node hashes to determine which are to be transferred.

			




	
	
	
		
 
	#

			




	
	
		
 
@@ -50,7 +56,7 @@ class Client:

			




	
	
	
		
 
	#

			




	
	
	
		
 
	# Requests nodes in order of a batch DFS. Needs stack of size O(treeDepth*batchSize). Nodes in each tree level are accessed in order.

			




	
	
	
		
 
	def negotiate(self):

			




	
	
	
		
 
		localTree=self._localTree

			




	
	
	
		
 
		localTree=self._tree

			




	
	
	
		
 
		blocksToTransfer=[]

			




	
	
	
		
 
		nodeStack=collections.deque([0]) # root

			




	
	
	
		
 

			




	
	
		
 
@@ -135,6 +141,9 @@ class Client:

			




	
	
	
		
 
				dataFile.seek(i2*HashTree.BLOCK_SIZE)

			




	
	
	
		
 
			dataFile.write(binData)

			




	
	
	
		
 

			




	
	
	
		
 
			if self._treeFile:

			




	
	
	
		
 
				self._newLeaves[i2+self._tree.leafStart]=hashBlock(binData)

			




	
	
	
		
 

			




	
	
	
		
 
			log.info("block #{0}: {1}...{2}".format(i2,binData[:5],binData[-5:]))

			




	
	
	
		
 

			




	
	
	
		
 
			stats.logTransferredBlock()

			




	
	
		
 
@@ -147,5 +156,11 @@ class Client:

			




	
	
	
		
 
		log.info("closing session...")

			




	
	
	
		
 
		dataFile.close()

			




	
	
	
		
 

			




	
	
	
		
 
		if self._treeFile:

			




	
	
	
		
 
			log.info("updating hash tree...")

			




	
	
	
		
 
			for (k,v) in self._newLeaves.items():

			




	
	
	
		
 
				self._tree.updateLeaf(k, v)

			




	
	
	
		
 
			self._tree.save(self._treeFile)

			




	
	
	
		
 

			




	
	
	
		
 
	def setConnection(self,connection):

			




	
	
	
		
 
		(self._incoming,self._outcoming)=connection

			




        

    


    
    

    

        

                

                    src/hashtree.py
                

                

                  
                      
                        

                      ➞

                      
                        
                  

                  
                      
                  
                      
                  
                      
                  
                      
                  
                  
                

                

                    Show inline comments
                    
                

        

        

            



	
	
		
 
@@ -5,6 +5,10 @@ from datetime import datetime

			




	
	
	
		
 
from util import Progress

			




	
	
	
		
 

			




	
	
	
		
 

			




	
	
	
		
 
def hashBlock(data):

			




	
	
	
		
 
	return hashlib.sha256(data).digest()[-HashTree.HASH_LEN:]

			




	
	
	
		
 

			




	
	
	
		
 

			




	
	
	
		
 
class HashTree:

			




	
	
	
		
 
	HASH_LEN=16 # bytes

			




	
	
	
		
 
	BLOCK_SIZE=4096 # bytes

			




	
	
		
 
@@ -28,7 +32,7 @@ class HashTree:

			




	
	
	
		
 
			progress=Progress(leafCount)

			




	
	
	
		
 
			for i in range(leafCount):

			




	
	
	
		
 
				data=f.read(HashTree.BLOCK_SIZE)

			




	
	
	
		
 
				res.insertLeaf(hashlib.sha256(data).digest()[-HashTree.HASH_LEN:])

			




	
	
	
		
 
				res.insertLeaf(hashBlock(data))

			




	
	
	
		
 

			




	
	
	
		
 
				progress.p(i)

			




	
	
	
		
 
			progress.done()

			




	
	
		
 
@@ -70,7 +74,7 @@ class HashTree:

			




	
	
	
		
 
	## Updates the node at index and all its ancestors.

			




	
	
	
		
 
	def updateNode(self,index):

			




	
	
	
		
 
		while index>=0:

			




	
	
	
		
 
			self.store[index]=hashlib.sha256(self.store[index*2+1]+self.store[index*2+2]).digest()[-HashTree.HASH_LEN:]

			




	
	
	
		
 
			self.store[index]=hashBlock(self.store[index*2+1]+self.store[index*2+2])

			




	
	
	
		
 
			index=(index-1)//2

			




	
	
	
		
 
			

			




	
	
	
		
 
	## Fast construction of the tree over the leaves. O(n).

			




	
	
		
 
@@ -78,14 +82,6 @@ class HashTree:

			




	
	
	
		
 
		print(datetime.now(), "building tree:")

			




	
	
	
		
 
		progress=Progress(-1, self.leafStart-1)

			




	
	
	
		
 
		for i in range(self.leafStart-1,-1,-1):

			




	
	
	
		
 
			self.store[i]=hashlib.sha256(self.store[i*2+1]+self.store[i*2+2]).digest()[-HashTree.HASH_LEN:]

			




	
	
	
		
 
			self.store[i]=hashBlock(self.store[i*2+1]+self.store[i*2+2])

			




	
	
	
		
 
			progress.p(i)

			




	
	
	
		
 
		progress.done()

			




	
	
	
		
 

			




	
	
	
		
 

			




	
	
	
		
 
if __name__=="__main__":

			




	
	
	
		
 
	f1=HashTree.fromFile(open("serverFile.txt",mode='rb'))

			




	
	
	
		
 
	f2=HashTree.fromFile(open("clientFile.txt",mode='rb'))

			




	
	
	
		
 

			




	
	
	
		
 
	for i,(h1,h2) in enumerate(zip(f1.store,f2.store)):

			




	
	
	
		
 
		print("{0:2}".format(i),h1.hex(),h2.hex(),h1==h2)

			




        

    


    
    

    

        

                

                    src/morevna.py
                

                

                  
                      
                        

                      ➞

                      
                        
                  

                  
                      
                  
                      
                  
                      
                  
                      
                  
                  
                

                

                    Show inline comments
                    
                

        

        

            



	
	
		
 
@@ -25,10 +25,12 @@ def buildTree(args):

			




	
	
	
		
 

			




	
	
	
		
 
def push(args):

			




	
	
	
		
 
	_checkFile(args.datafile)

			




	
	
	
		
 
	if args.tree:

			




	
	
	
		
 
		_checkFile(args.tree)

			




	
	
	
		
 
	if args.host: conf.hosts.insert(0,args.host)

			




	
	
	
		
 
	if args.port: conf.port=args.port

			




	
	
	
		
 

			




	
	
	
		
 
	c=Client(args.datafile)

			




	
	
	
		
 
	c=Client(args.datafile,args.tree)

			




	
	
	
		
 
	with ClientConnection() as con:

			




	
	
	
		
 
		c.setConnection(con)

			




	
	
	
		
 
		blocksToTransfer=c.negotiate()

			




	
	
		
 
@@ -38,10 +40,12 @@ def push(args):

			




	
	
	
		
 

			




	
	
	
		
 
def pull(args):

			




	
	
	
		
 
	_checkFile(args.datafile)

			




	
	
	
		
 
	if args.tree:

			




	
	
	
		
 
		_checkFile(args.tree)

			




	
	
	
		
 
	if args.host: conf.hosts.insert(0,args.host)

			




	
	
	
		
 
	if args.port: conf.port=args.port

			




	
	
	
		
 

			




	
	
	
		
 
	c=Client(args.datafile)

			




	
	
	
		
 
	c=Client(args.datafile,args.tree)

			




	
	
	
		
 
	with ClientConnection() as con:

			




	
	
	
		
 
		c.setConnection(con)

			




	
	
	
		
 
		blocksToTransfer=c.negotiate()

			




	
	
		
 
@@ -74,12 +78,14 @@ pBuild.set_defaults(func=buildTree)

			




	
	
	
		
 
pUpdate=subparsers.add_parser("push")

			




	
	
	
		
 
pUpdate.add_argument("-p","--port",type=int)

			




	
	
	
		
 
pUpdate.add_argument("--host",default="127.0.0.1")

			




	
	
	
		
 
pUpdate.add_argument("-t","--tree",help="stored hash tree location")

			




	
	
	
		
 
pUpdate.add_argument("datafile")

			




	
	
	
		
 
pUpdate.set_defaults(func=push)

			




	
	
	
		
 

			




	
	
	
		
 
pUpdate=subparsers.add_parser("pull")

			




	
	
	
		
 
pUpdate.add_argument("-p","--port",type=int)

			




	
	
	
		
 
pUpdate.add_argument("--host",default="127.0.0.1")

			




	
	
	
		
 
pUpdate.add_argument("-t","--tree",help="stored hash tree location")

			




	
	
	
		
 
pUpdate.add_argument("datafile")

			




	
	
	
		
 
pUpdate.set_defaults(func=pull)

			




	
	
	
		
 

			




        

    


    
    

    

        

                

                    src/server.py
                

                

                  
                      
                        

                      ➞

                      
                        
                  

                  
                      
                  
                      
                  
                      
                  
                      
                  
                  
                

                

                    Show inline comments
                    
                

        

        

            



	
	
	
		
 
import hashlib

			




	
	
	
		
 
import socket

			




	
	
	
		
 
import socket

			




	
	
	
		
 
import ssl

			




	
	
	
		
 
import multiprocessing

			




	
	
	
		
 
import logging as log

			




	
	
	
		
 

			




	
	
	
		
 
from hashtree import HashTree

			




	
	
	
		
 
from hashtree import HashTree,hashBlock

			




	
	
	
		
 
from networkers import NetworkReader,NetworkWriter

			




	
	
	
		
 
import config as conf

			




	
	
	
		
 

			




	
	
		
 
@@ -140,7 +139,7 @@ class Server:

			




	
	
	
		
 
		self._dataFile.write(binData)

			




	
	
	
		
 
		self._lastIndex=i

			




	
	
	
		
 
		if self._treeFile:

			




	
	
	
		
 
			self._newLeaves[i+self._tree.leafStart]=hashlib.sha256(binData).digest()[HashTree.HASH_LEN:]

			




	
	
	
		
 
			self._newLeaves[i+self._tree.leafStart]=hashBlock(binData)

			




	
	
	
		
 

			




	
	
	
		
 
		return ({"command": "ack", "index": i},)

			




	
	
	
		
 

			




        

    



    


    



    



      

      





    
    0 comments (0 inline, 0 general)
    





    


  

  







  


    




    








    
        Server instance: Sirius-537
    
    
        This site is powered by
            Kallithea 0.7.0,
        which is
        © 2010–2023 by various authors & licensed under GPLv3.
            – Support