Changeset - b052f27e1cbc
[Not reviewed]
default
0 4 0
Laman - 8 years ago 2017-06-03 22:47:05

more progress info
4 files changed with 29 insertions and 13 deletions:
0 comments (0 inline, 0 general)
src/client.py
Show inline comments
 
from hashtree import HashTree
 
import collections
 
import collections
 
import socket
 
import logging as log
 
from datetime import datetime
 

	
 
import config as conf
 
from util import progress
 
from hashtree import HashTree
 
from networkers import NetworkReader,NetworkWriter
 

	
 

	
 
class Connection:
 
	def __init__(self):
 
		self.socket=socket.socket(socket.AF_INET,socket.SOCK_STREAM)
 
		self.socket.connect((conf.hosts[0], conf.port))
 
		fr=self.socket.makefile(mode="rb")
 
		fw=self.socket.makefile(mode="wb")
 

	
 
		self.incoming=NetworkReader(fr)
 
		self.outcoming=NetworkWriter(fw)
 
@@ -20,61 +22,69 @@ class Connection:
 
	def __enter__(self):
 
		return self.incoming,self.outcoming
 

	
 
	def __exit__(self, exc_type, exc_val, exc_tb):
 
		self.socket.close()
 

	
 

	
 
class Client:
 
	def __init__(self,filename):
 
		self.filename=filename
 

	
 
	def negotiate(self):
 
		print(datetime.now(), "initializing...")
 
		localTree=HashTree.fromFile(self.filename)
 
		blocksToTransfer=[]
 
		nodeStack=collections.deque([0]) # root
 

	
 
		# initialize session
 
		with Connection() as (incoming,outcoming):
 
			jsonData={"command":"init", "blockSize":localTree.BLOCK_SIZE, "blockCount":localTree.leafCount, "version":conf.version}
 
			outcoming.writeMsg(jsonData)
 

	
 
		# determine which blocks to send
 
		print(datetime.now(), "negotiating:")
 
		while len(nodeStack)>0:
 
			with Connection() as (incoming,outcoming):
 
				i=nodeStack.pop()
 
				outcoming.writeMsg({"command":"req", "index":i})
 

	
 
				jsonData,binData=incoming.readMsg()
 
				assert jsonData["index"]==i
 
				assert jsonData["dataType"]=="hash"
 

	
 
				if localTree.store[i]!=binData:
 
					if 2*i+3<len(localTree.store): # inner node
 
						nodeStack.append(2*i+2)
 
						nodeStack.append(2*i+1)
 
					else: blocksToTransfer.append(i-localTree.leafStart) # leaf
 
					else:
 
						blocksToTransfer.append(i-localTree.leafStart) # leaf
 
						progress(i-localTree.leafStart, localTree.leafCount)
 
		print("100%")
 

	
 
		return blocksToTransfer
 

	
 
	def sendData(self,blocksToTransfer):
 
		log.info(blocksToTransfer)
 
		dataFile=open(self.filename,mode="rb")
 
		i1=-1
 

	
 
		for i2 in blocksToTransfer:
 
		print(datetime.now(), "sending data:")
 
		for (k,i2) in enumerate(blocksToTransfer):
 
			with Connection() as (incoming,outcoming):
 
				jsonData={"command":"send", "index":i2, "dataType":"data"}
 
				if i1+1!=i2:
 
					dataFile.seek(i2*HashTree.BLOCK_SIZE)
 
				binData=dataFile.read(HashTree.BLOCK_SIZE)
 

	
 
				log.info("block #{0}: {1}...{2}".format(i2,binData[:5],binData[-5:]))
 

	
 
				outcoming.writeMsg(jsonData,binData)
 
				i1=i2
 
			i1=i2
 
			progress(k,len(blocksToTransfer))
 
		print("100%")
 

	
 
		with Connection() as (incoming,outcoming):
 
			outcoming.writeMsg({"command":"end"})
 

	
 
		log.info("closing session...")
 
		log.info(datetime.now(), "closing session...")
 
		dataFile.close()
src/config.py
Show inline comments
 
import datetime
 
import logging as log
 

	
 

	
 
log.basicConfig(level=log.INFO,format="%(asctime)s %(levelname)s: %(message)s",datefmt="%Y-%m-%d %H:%M:%S")
 
logName="/tmp/mor{0}.log".format(datetime.datetime.now().timestamp())
 
log.basicConfig(
 
	level=log.INFO,format="%(asctime)s %(levelname)s: %(message)s",datefmt="%Y-%m-%d %H:%M:%S",
 
	filename=logName
 
)
 

	
 
version=0
 

	
 
hosts=["127.0.0.1"]
 
port=50009
src/hashtree.py
Show inline comments
 
import hashlib
 
import os
 
from datetime import datetime
 

	
 
from util import progress
 

	
 

	
 
class HashTree:
 
	HASH_LEN=16 # bytes
 
	BLOCK_SIZE=4096 # bytes
 
	
 
	## Prepares a tree containing leafCount leaves.
 
	def __init__(self,leafCount):
 
		self.store=[b""]*(leafCount*2-1)
 
		self.leafStart=leafCount-1
 
		self.index=self.leafStart
 
		self.leafCount=leafCount
 
		
 
	@classmethod
 
	def fromFile(cls,filename):
 
		with open(filename,"rb") as f:
 
			stat=os.fstat(f.fileno())
 
			size=stat.st_size # !! symlinks
 
			leafCount=(size-1)//HashTree.BLOCK_SIZE+1 # number of leaf blocks
 
			res=cls(leafCount)
 
			print("hashing file:")
 
			print(datetime.now(), "hashing file:")
 

	
 
			for i in range(leafCount):
 
				data=f.read(HashTree.BLOCK_SIZE)
 
				res.insertLeaf(hashlib.sha256(data).digest()[HashTree.HASH_LEN:])
 

	
 
				progress(i, leafCount)
 
			print("100%")
 
		res.buildTree()
 
		
 
		return res
 

	
 
	@classmethod
 
	def load(cls,filename):
 
		with open(filename,"rb") as f:
 
			stat=os.fstat(f.fileno())
 
			size=stat.st_size
 
			nodeCount=size//HashTree.HASH_LEN
 
			res=cls((nodeCount+1)//2)
 

	
 
@@ -64,24 +66,25 @@ class HashTree:
 
		
 
		self.store[index]=h
 
		self.updateNode((index-1)//2)
 
	
 
	## Updates the node at index and all its ancestors.
 
	def updateNode(self,index):
 
		while index>=0:
 
			self.store[index]=hashlib.sha256(self.store[index*2+1]+self.store[index*2+2]).digest()[HashTree.HASH_LEN:]
 
			index=(index-1)//2
 
			
 
	## Fast construction of the tree over the leaves. O(n).
 
	def buildTree(self):
 
		print("building tree:")
 
		print(datetime.now(), "building tree:")
 
		for i in range(self.leafStart-1,-1,-1):
 
			self.store[i]=hashlib.sha256(self.store[i*2+1]+self.store[i*2+2]).digest()[HashTree.HASH_LEN:]
 
			progress(i, -1, self.leafStart - 1)
 
		print()
 

	
 

	
 
if __name__=="__main__":
 
	f1=HashTree.fromFile(open("serverFile.txt",mode='rb'))
 
	f2=HashTree.fromFile(open("clientFile.txt",mode='rb'))
 

	
 
	for i,(h1,h2) in enumerate(zip(f1.store,f2.store)):
 
		print("{0:2}".format(i),h1.hex(),h2.hex(),h1==h2)
src/util.py
Show inline comments
 
def progress(i, n, i0=0):
 
	def _progress(i,n,i0):
 
		return 100*(i+1-i0)//(n-i0)
 

	
 
	if n<i0:
 
		i*=-1
 
		n*=-1
 
		i0*=-1
 
	assert i0<=i<n
 
	assert i0<=i<n, (i0,i,n)
 
	percentage=_progress(i,n,i0)
 
	if percentage>_progress(i-1,n,i0):
 
		print(r"{0}%".format(percentage),end="")
 
		if percentage==100: print()
 
	print("{0}%".format(percentage),end="\r")
0 comments (0 inline, 0 general)