Changeset - 4b88aca70fbc
[Not reviewed]
default
0 3 0
Laman - 8 years ago 2017-06-12 21:29:32

Progress object
3 files changed with 38 insertions and 20 deletions:
0 comments (0 inline, 0 general)
src/client.py
Show inline comments
 
import collections
 
import socket
 
import logging as log
 
from datetime import datetime
 

	
 
import config as conf
 
from util import progress
 
from util import Progress
 
from hashtree import HashTree
 
from networkers import NetworkReader,NetworkWriter
 

	
 

	
 
class Connection:
 
	def __init__(self):
 
@@ -42,12 +42,13 @@ class Client:
 
			outcoming.writeMsg(jsonData)
 
			jsonData,binData=incoming.readMsg()
 
			assert jsonData["command"]=="ack"
 

	
 
			# determine which blocks to send
 
			print(datetime.now(), "negotiating:")
 
			progress=Progress(localTree.leafCount)
 
			while len(nodeStack)>0:
 
				i=nodeStack.pop()
 
				outcoming.writeMsg({"command":"req", "index":i})
 

	
 
				jsonData,binData=incoming.readMsg()
 
				assert jsonData["index"]==i
 
@@ -56,38 +57,39 @@ class Client:
 
				if localTree.store[i]!=binData:
 
					if 2*i+3<len(localTree.store): # inner node
 
						nodeStack.append(2*i+2)
 
						nodeStack.append(2*i+1)
 
					else:
 
						blocksToTransfer.append(i-localTree.leafStart) # leaf
 
						progress(i-localTree.leafStart, localTree.leafCount)
 
		print("100%")
 
						progress.p(i-localTree.leafStart)
 
		progress.done()
 

	
 
		return blocksToTransfer
 

	
 
	def sendData(self,blocksToTransfer):
 
		log.info(blocksToTransfer)
 
		dataFile=open(self.filename,mode="rb")
 
		i1=-1
 

	
 
		print(datetime.now(), "sending data:")
 
		with Connection() as (incoming,outcoming):
 
			progress=Progress(len(blocksToTransfer))
 
			for (k,i2) in enumerate(blocksToTransfer):
 
				jsonData={"command":"send", "index":i2, "dataType":"data"}
 
				if i1+1!=i2:
 
					dataFile.seek(i2*HashTree.BLOCK_SIZE)
 
				binData=dataFile.read(HashTree.BLOCK_SIZE)
 

	
 
				log.info("block #{0}: {1}...{2}".format(i2,binData[:5],binData[-5:]))
 

	
 
				outcoming.writeMsg(jsonData,binData)
 
				jsonData,binData=incoming.readMsg()
 
				assert jsonData["command"]=="ack" and jsonData["index"]==i2, jsonData
 
				i1=i2
 
				progress(k,len(blocksToTransfer))
 
		print("100%")
 
				progress.p(k)
 
		progress.done()
 

	
 
		with Connection() as (incoming,outcoming):
 
			outcoming.writeMsg({"command":"end"})
 

	
 
		log.info("closing session...")
 
		dataFile.close()
src/hashtree.py
Show inline comments
 
import hashlib
 
import os
 
from datetime import datetime
 

	
 
from util import progress
 
from util import Progress
 

	
 

	
 
class HashTree:
 
	HASH_LEN=16 # bytes
 
	BLOCK_SIZE=4096 # bytes
 
	
 
@@ -22,18 +22,19 @@ class HashTree:
 
			stat=os.fstat(f.fileno())
 
			size=stat.st_size # !! symlinks
 
			leafCount=(size-1)//HashTree.BLOCK_SIZE+1 # number of leaf blocks
 
			res=cls(leafCount)
 
			print(datetime.now(), "hashing file:")
 

	
 
			progress=Progress(leafCount)
 
			for i in range(leafCount):
 
				data=f.read(HashTree.BLOCK_SIZE)
 
				res.insertLeaf(hashlib.sha256(data).digest()[HashTree.HASH_LEN:])
 

	
 
				progress(i, leafCount)
 
			print("100%")
 
				progress.p(i)
 
			progress.done()
 
		res.buildTree()
 
		
 
		return res
 

	
 
	@classmethod
 
	def load(cls,filename):
 
@@ -73,16 +74,17 @@ class HashTree:
 
			self.store[index]=hashlib.sha256(self.store[index*2+1]+self.store[index*2+2]).digest()[HashTree.HASH_LEN:]
 
			index=(index-1)//2
 
			
 
	## Fast construction of the tree over the leaves. O(n).
 
	def buildTree(self):
 
		print(datetime.now(), "building tree:")
 
		progress=Progress(-1, self.leafStart-1)
 
		for i in range(self.leafStart-1,-1,-1):
 
			self.store[i]=hashlib.sha256(self.store[i*2+1]+self.store[i*2+2]).digest()[HashTree.HASH_LEN:]
 
			progress(i, -1, self.leafStart - 1)
 
		print()
 
			progress.p(i)
 
		progress.done()
 

	
 

	
 
if __name__=="__main__":
 
	f1=HashTree.fromFile(open("serverFile.txt",mode='rb'))
 
	f2=HashTree.fromFile(open("clientFile.txt",mode='rb'))
 

	
src/util.py
Show inline comments
 
def progress(i, n, i0=0):
 
	def _progress(i,n,i0):
 
		return 100*(i+1-i0)//(n-i0)
 
class Progress:
 
	def __init__(self,n,i0=0):
 
		self._n=n
 
		self._i0=i0
 
		self._i=i0
 
		self._last=""
 

	
 
	def p(self,i):
 
		i0=self._i0
 
		n=self._n
 

	
 
	if n<i0:
 
		i*=-1
 
		n*=-1
 
		i0*=-1
 
	assert i0<=i<n, (i0,i,n)
 
	percentage=_progress(i,n,i0)
 
	print("{0}%".format(percentage),end="\r")
 
		assert i0<=i<n or n<i<=i0, (i0,i,n)
 
		percentage=Progress._p(i,n,i0)
 
		res="{0}%".format(percentage)
 
		if res!=self._last:
 
			print(res,end="\r")
 
			self._last=res
 

	
 
	def done(self):
 
		print("100%")
 

	
 
	@staticmethod
 
	def _p(i,n,i0):
 
		_1=1 if n>=i0 else -1
 
		return 100*(i+_1-i0)//(n-i0)
0 comments (0 inline, 0 general)