Changeset - 13d0327a4abb
[Not reviewed]
default
0 3 1
Laman - 8 years ago 2017-05-08 13:16:05

rudimentary progress info
4 files changed with 24 insertions and 5 deletions:
0 comments (0 inline, 0 general)
src/client.py
Show inline comments
 
from hashtree import HashTree
 
import collections
 
import socket
 
import sys
 
import logging as log
 

	
 
import config as conf
 
from networkers import NetworkReader,NetworkWriter
 

	
 

	
 
class Connection:
 
	def __init__(self):
 
		self.socket=socket.socket(socket.AF_INET,socket.SOCK_STREAM)
 
		self.socket.connect((conf.hosts[0], conf.port))
 
		fr=self.socket.makefile(mode="rb")
 
		fw=self.socket.makefile(mode="wb")
 

	
 
		self.incoming=NetworkReader(fr)
 
		self.outcoming=NetworkWriter(fw)
 

	
 
	def __enter__(self):
 
		return self.incoming,self.outcoming
 

	
 
	def __exit__(self, exc_type, exc_val, exc_tb):
 
		self.socket.close()
 

	
 

	
 
class Client:
 
	def __init__(self,filename):
 
		self.filename=filename
 

	
 
	def negotiate(self):
 
		localTree=HashTree.fromFile(self.filename)
 
		blocksToTransfer=[]
 
		nodeStack=collections.deque([0]) # root
 

	
 
		# initialize session
 
		with Connection() as (incoming,outcoming):
 
			jsonData={"command":"init", "blockSize":localTree.BLOCK_SIZE, "blockCount":localTree.leafCount, "version":conf.version}
 
			outcoming.writeMsg(jsonData)
 

	
 
		# determine which blocks to send
 
		while len(nodeStack)>0:
 
			with Connection() as (incoming,outcoming):
 
				i=nodeStack.pop()
 
				outcoming.writeMsg({"command":"req", "index":i})
 

	
 
				jsonData,binData=incoming.readMsg()
 
				assert jsonData["index"]==i
 
				assert jsonData["dataType"]=="hash"
 

	
 
				if localTree.store[i]!=binData:
src/hashtree.py
Show inline comments
 
import hashlib
 
import os
 

	
 
from util import progress
 

	
 

	
 
class HashTree:
 
	HASH_LEN=16 # bytes
 
	BLOCK_SIZE=4096 # bytes
 
	
 
	## Prepares a tree containing leafCount leaves.
 
	def __init__(self,leafCount):
 
		self.store=[b""]*(leafCount*2-1)
 
		self.leafStart=leafCount-1
 
		self.index=self.leafStart
 
		self.leafCount=leafCount
 
		
 
	@classmethod
 
	def fromFile(cls,filename):
 
		with open(filename,"rb") as f:
 
			stat=os.fstat(f.fileno())
 
			size=stat.st_size # !! symlinks
 
			leafCount=(size-1)//HashTree.BLOCK_SIZE+1 # number of leaf blocks
 
			res=cls(leafCount)
 
			print("hashing file:")
 

	
 
			for i in range(leafCount):
 
				data=f.read(HashTree.BLOCK_SIZE)
 
				res.insertLeaf(hashlib.sha256(data).digest()[HashTree.HASH_LEN:])
 

	
 
				progress(i, leafCount)
 
		res.buildTree()
 
		
 
		return res
 

	
 
	@classmethod
 
	def load(cls,filename):
 
		with open(filename,"rb") as f:
 
			stat=os.fstat(f.fileno())
 
			size=stat.st_size
 
			nodeCount=size//HashTree.HASH_LEN
 
			res=cls((nodeCount+1)//2)
 

	
 
			for i in range(nodeCount):
 
				res.store[i]=f.read(HashTree.HASH_LEN)
 
		return res
 

	
 
	def save(self,filename):
 
		with open(filename,"wb") as f:
 
			for h in self.store:
 
				f.write(h)
 

	
 
		
 
	## Inserts a leaf at the first empty position.
 
	#	
 
	#	Useful and used only during the tree construction.
 
	def insertLeaf(self,h):
 
		self.store[self.index]=h
 
		self.index+=1
 
		
 
	## Updates a hash stored in the leaf.
 
	def updateLeaf(self,index,h):
 
		if index<self.leafStart: raise IndexError()
 
		
 
		self.store[index]=h
 
		self.updateNode((index-1)//2)
 
	
 
	## Updates the node at index and all its ancestors.
 
	def updateNode(self,index):
 
		while index>=0:
 
			self.store[index]=hashlib.sha256(self.store[index*2+1]+self.store[index*2+2]).digest()[HashTree.HASH_LEN:]
 
			index=(index-1)//2
 
			
 
	## Fast construction of the tree over the leaves. O(n).
 
	def buildTree(self):
 
		print("building tree:")
 
		for i in range(self.leafStart-1,-1,-1):
 
			self.store[i]=hashlib.sha256(self.store[i*2+1]+self.store[i*2+2]).digest()[HashTree.HASH_LEN:]
 
			progress(i, -1, self.leafStart - 1)
 

	
 

	
 
if __name__=="__main__":
 
	f1=HashTree.fromFile(open("serverFile.txt",mode='rb'))
 
	f2=HashTree.fromFile(open("clientFile.txt",mode='rb'))
 

	
 
	for i,(h1,h2) in enumerate(zip(f1.store,f2.store)):
 
		print("{0:2}".format(i),h1.hex(),h2.hex(),h1==h2)
src/morevna.py
Show inline comments
 
import sys
 
import os.path
 
from argparse import ArgumentParser
 

	
 
from hashtree import HashTree
 
from client import Client
 
from server import Server
 

	
 

	
 
def _checkFile(f):
 
	if not os.path.isfile(f):
 
		print("invalid file specified:",f,file=sys.stderr)
 
		sys.exit(1)
 

	
 

	
 
def buildTree(args):
 
	_checkFile(args.datafile)
 

	
 
	tree=HashTree.fromFile(args.datafile)
 
	tree.save(args.treefile)
 

	
 
def update(args):
 
	_checkFile(args.datafile)
 

	
 
	c=Client(args.datafile)
 
	blocksToTransfer=c.negotiate()
 
	c.sendData(blocksToTransfer)
 

	
 
def serve(args):
 
	_checkFile(args.datafile)
 
	if args.tree:
 
		_checkFile(args.tree)
 

	
 
	# debug copy default file
 
	import shutil
 
	origFilename=args.datafile
 
	filename=origFilename+"_"
 
	shutil.copyfile(origFilename,filename)
 

	
 
	s=Server(filename,args.tree)
 
	s.serve()
 

	
 

	
 
parser=ArgumentParser()
 
subparsers=parser.add_subparsers()
 

	
 
pRebuild=subparsers.add_parser("build")
 
pRebuild.add_argument("treefile",help="stored hash tree location")
 
pRebuild.add_argument("datafile")
 
pRebuild.set_defaults(func=buildTree)
 
pBuild=subparsers.add_parser("build")
 
pBuild.add_argument("treefile", help="stored hash tree location")
 
pBuild.add_argument("datafile")
 
pBuild.set_defaults(func=buildTree)
 

	
 
pUpdate=subparsers.add_parser("update")
 
pUpdate.add_argument("-p","--port",type=int)
 
pUpdate.add_argument("--host",default="127.0.0.1")
 
pUpdate.add_argument("-t","--tree",help="stored hash tree location")
 
pUpdate.add_argument("datafile")
 
pUpdate.set_defaults(func=update)
 

	
 
pServe=subparsers.add_parser("serve")
 
pServe.add_argument("-p","--port",type=int)
 
pServe.add_argument("--host",default="")
 
pServe.add_argument("-t","--tree",help="stored hash tree location")
 
pServe.add_argument("datafile")
 
pServe.set_defaults(func=serve)
 

	
 
args=parser.parse_args()
 
args.func(args)
src/util.py
Show inline comments
 
new file 100644
 
def progress(i, n, i0=0):
 
	def _progress(i,n,i0):
 
		return 100*(i+1-i0)//(n-i0)
 

	
 
	if n<i0:
 
		i*=-1
 
		n*=-1
 
		i0*=-1
 
	assert i0<=i<n
 
	percentage=_progress(i,n,i0)
 
	if percentage>_progress(i-1,n,i0):
 
		print(r"{0}%".format(percentage),end="")
 
		if percentage==100: print()
0 comments (0 inline, 0 general)