Changeset - 41ea9614ce8c
[Not reviewed]
default
0 3 1
Laman - 8 years ago 2017-06-13 22:23:30

collecting basic statistics
4 files changed with 46 insertions and 1 deletions:
0 comments (0 inline, 0 general)
src/client.py
Show inline comments
 
import collections
 
import socket
 
import logging as log
 
from datetime import datetime
 

	
 
import config as conf
 
import stats
 
from util import Progress
 
from hashtree import HashTree
 
from networkers import NetworkReader,NetworkWriter
 

	
 

	
 
class Connection:
 
	def __init__(self):
 
		self.socket=socket.socket(socket.AF_INET,socket.SOCK_STREAM)
 
		self.socket.connect((conf.hosts[0], conf.port))
 
		fr=self.socket.makefile(mode="rb")
 
		fw=self.socket.makefile(mode="wb")
 

	
 
		self.incoming=NetworkReader(fr)
 
		self.outcoming=NetworkWriter(fw)
 

	
 
	def __enter__(self):
 
		return self.incoming,self.outcoming
 

	
 
	def __exit__(self, exc_type, exc_val, exc_tb):
 
		self.socket.close()
 

	
 

	
 
class Client:
 
	def __init__(self,filename):
 
@@ -32,64 +33,66 @@ class Client:
 

	
 
	def negotiate(self):
 
		print(datetime.now(), "initializing...")
 
		localTree=HashTree.fromFile(self.filename)
 
		blocksToTransfer=[]
 
		nodeStack=collections.deque([0]) # root
 

	
 
		# initialize session
 
		with Connection() as (incoming,outcoming):
 
			jsonData={"command":"init", "blockSize":localTree.BLOCK_SIZE, "blockCount":localTree.leafCount, "version":conf.version}
 
			outcoming.writeMsg(jsonData)
 
			jsonData,binData=incoming.readMsg()
 
			assert jsonData["command"]=="ack"
 

	
 
			# determine which blocks to send
 
			print(datetime.now(), "negotiating:")
 
			progress=Progress(localTree.leafCount)
 
			while len(nodeStack)>0:
 
				i=nodeStack.pop()
 
				outcoming.writeMsg({"command":"req", "index":i})
 

	
 
				jsonData,binData=incoming.readMsg()
 
				assert jsonData["index"]==i
 
				assert jsonData["dataType"]=="hash"
 
				stats.logExchangedNode()
 

	
 
				if localTree.store[i]!=binData:
 
					if 2*i+3<len(localTree.store): # inner node
 
						nodeStack.append(2*i+2)
 
						nodeStack.append(2*i+1)
 
					else:
 
						blocksToTransfer.append(i-localTree.leafStart) # leaf
 
						progress.p(i-localTree.leafStart)
 
		progress.done()
 

	
 
		return blocksToTransfer
 

	
 
	def sendData(self,blocksToTransfer):
 
		log.info(blocksToTransfer)
 
		dataFile=open(self.filename,mode="rb")
 
		i1=-1
 

	
 
		print(datetime.now(), "sending data:")
 
		with Connection() as (incoming,outcoming):
 
			progress=Progress(len(blocksToTransfer))
 
			for (k,i2) in enumerate(blocksToTransfer):
 
				jsonData={"command":"send", "index":i2, "dataType":"data"}
 
				if i1+1!=i2:
 
					dataFile.seek(i2*HashTree.BLOCK_SIZE)
 
				binData=dataFile.read(HashTree.BLOCK_SIZE)
 

	
 
				log.info("block #{0}: {1}...{2}".format(i2,binData[:5],binData[-5:]))
 

	
 
				outcoming.writeMsg(jsonData,binData)
 
				stats.logTransferredBlock()
 
				jsonData,binData=incoming.readMsg()
 
				assert jsonData["command"]=="ack" and jsonData["index"]==i2, jsonData
 
				i1=i2
 
				progress.p(k)
 
		progress.done()
 

	
 
		with Connection() as (incoming,outcoming):
 
			outcoming.writeMsg({"command":"end"})
 

	
 
		log.info("closing session...")
 
		dataFile.close()
src/morevna.py
Show inline comments
 
import sys
 
import os.path
 
from argparse import ArgumentParser
 

	
 
import config as conf
 
import stats
 
from hashtree import HashTree
 
from client import Client
 
from server import Server
 

	
 

	
 
def _checkFile(f):
 
	if not os.path.isfile(f):
 
		print("invalid file specified:",f,file=sys.stderr)
 
		sys.exit(1)
 

	
 

	
 
def buildTree(args):
 
	_checkFile(args.datafile)
 

	
 
	tree=HashTree.fromFile(args.datafile)
 
	tree.save(args.treefile)
 

	
 
def update(args):
 
	_checkFile(args.datafile)
 
	if args.host: conf.hosts.insert(0,args.host)
 
	if args.port: conf.port=args.port
 

	
 
	c=Client(args.datafile)
 
	blocksToTransfer=c.negotiate()
 
	c.sendData(blocksToTransfer)
 
	print()
 
	print(stats.report())
 

	
 
def serve(args):
 
	_checkFile(args.datafile)
 
	if args.tree:
 
		_checkFile(args.tree)
 
	if args.host: conf.hosts.insert(0,args.host)
 
	if args.port: conf.port=args.port
 

	
 
	s=Server(args.datafile,args.tree)
 
	s.serve()
 

	
 

	
 
parser=ArgumentParser()
 
subparsers=parser.add_subparsers()
 

	
 
pBuild=subparsers.add_parser("build")
 
pBuild.add_argument("treefile", help="stored hash tree location")
 
pBuild.add_argument("datafile")
 
pBuild.set_defaults(func=buildTree)
 

	
 
pUpdate=subparsers.add_parser("update")
 
pUpdate.add_argument("-p","--port",type=int)
 
pUpdate.add_argument("--host",default="127.0.0.1")
 
pUpdate.add_argument("datafile")
src/networkers.py
Show inline comments
 
import json
 

	
 
import stats
 

	
 

	
 
class NetworkReader:
 
	def __init__(self,stream):
 
		self.stream=stream
 

	
 
	def readMsg(self):
 
		data=self.stream.readline()
 
		assert data
 
		stats.logReceived(data)
 
		jsonLength=int(data.split(b":")[1].strip()) # "json-length: length" -> length
 

	
 
		data=self.stream.readline()
 
		assert data
 
		stats.logReceived(data)
 
		binLength=int(data.split(b":")[1].strip()) # "bin-length: length" -> length
 

	
 
		jsonData=self.stream.read(jsonLength)
 
		assert len(jsonData)==jsonLength
 
		stats.logReceived(jsonData)
 
		jsonData=json.loads(str(jsonData,encoding="utf-8"))
 

	
 
		binData=self.stream.read(binLength)
 
		assert len(binData)==binLength
 
		stats.logReceived(binData)
 
		
 
		return (jsonData,binData)
 
		
 

	
 
class NetworkWriter:
 
	def __init__(self,stream):
 
		self.stream=stream
 

	
 
	def writeMsg(self,*args):
 
		self.stream.write(self.prepMsg(*args))
 
		msg=self.prepMsg(*args)
 
		self.stream.write(msg)
 
		self.stream.flush()
 
		stats.logSent(msg)
 

	
 
	def prepMsg(self,jsonData,binData=b""):
 
		jsonData=bytes(json.dumps(jsonData)+"\n",encoding="utf-8")
 
		jsonLength=bytes("json-length: "+str(len(jsonData))+"\n",encoding="utf-8")
 
		binLength=bytes("bin-length: "+str(len(binData))+"\n",encoding="utf-8")
 

	
 
		return b"".join((jsonLength,binLength,jsonData,binData))
src/stats.py
Show inline comments
 
new file 100644
 
class Stats:
 
	received=0
 
	sent=0
 
	exchangedNodes=0
 
	transferredBlocks=0
 

	
 

	
 
def logReceived(data):
 
	Stats.received+=len(data)
 

	
 

	
 
def logSent(data):
 
	Stats.sent+=len(data)
 

	
 

	
 
def logExchangedNode():
 
	Stats.exchangedNodes+=1
 

	
 

	
 
def logTransferredBlock():
 
	Stats.transferredBlocks+=1
 

	
 

	
 
def report():
 
	return """received {r}B
 
sent {s}B
 
exchanged {nodes} hash tree nodes
 
transferred {blocks} blocks""".format(r=Stats.received, s=Stats.sent, nodes=Stats.exchangedNodes, blocks=Stats.transferredBlocks)
0 comments (0 inline, 0 general)