Changeset - fa1be84731e2
Parent rev.
Child rev.
[Not reviewed]
default
0 3 0
Laman - 8 years ago 2017-05-07 00:17:09

simplified networkers for dealing with a single request-response
3 files changed with 17 insertions and 47 deletions:
src/client.py
7
14
src/networkers.py
6
24
src/server.py
4
9
0 comments (0 inline, 0 general)
src/client.py
➞
Show inline comments
 
@@ -8,57 +8,51 @@ from networkers import NetworkReader,Net
 
 
 
filename=sys.argv[1]
 
 
 
class Connection:
 
	def __init__(self):
 
		self.socket=socket.socket(socket.AF_INET,socket.SOCK_STREAM)
 
		self.socket.connect((conf.hosts[0], conf.port))
 
		fr=self.socket.makefile(mode="rb")
 
		fw=self.socket.makefile(mode="wb")
 
 
		networkReader=NetworkReader(fr)
 
		networkReader.start()
 
		networkWriter=NetworkWriter(fw)
 
		networkWriter.start()
 
 
		self.incoming=networkReader.output # synchronized message queue
 
		self.outcoming=networkWriter.input
 
		self.incoming=NetworkReader(fr)
 
		self.outcoming=NetworkWriter(fw)
 
 
	def __enter__(self):
 
		return self.incoming,self.outcoming
 
 
	def __exit__(self, exc_type, exc_val, exc_tb):
 
		self.socket.close()
 
 
 
def negotiate():
 
	localTree=HashTree.fromFile(open(filename,mode="rb"))
 
	blocksToTransfer=[]
 
	nodeStack=collections.deque([0]) # root
 
 
	# initialize session
 
	with Connection() as (incoming,outcoming):
 
		jsonData={"command":"init", "blockSize":localTree.BLOCK_SIZE, "blockCount":localTree.leafCount, "version":conf.version}
 
		outcoming.put((jsonData,b""))
 
		outcoming.writeMsg(jsonData)
 
 
	# determine which blocks to send
 
	while len(nodeStack)>0:
 
		with Connection() as (incoming,outcoming):
 
			i=nodeStack.pop()
 
			jsonData={"command":"req", "index":i}
 
			outcoming.put((jsonData,b""))
 
			outcoming.writeMsg({"command":"req", "index":i})
 
 
			jsonData,binData=incoming.get(timeout=2)
 
			jsonData,binData=incoming.readMsg()
 
			assert jsonData["index"]==i
 
			assert jsonData["dataType"]=="hash"
 
 
			if localTree.store[i]!=binData:
 
				if 2*i+3<len(localTree.store): # inner node
 
					nodeStack.append(2*i+2)
 
					nodeStack.append(2*i+1)
 
				else: blocksToTransfer.append(i-localTree.leafStart) # leaf
 
 
	return blocksToTransfer
 
 
 
@@ -67,28 +61,27 @@ def sendData(blocksToTransfer):
 
	dataFile=open(filename,mode="rb")
 
	i1=-1
 
 
	for i2 in blocksToTransfer:
 
		with Connection() as (incoming,outcoming):
 
			jsonData={"command":"send", "index":i2, "dataType":"data"}
 
			if i1+1!=i2:
 
				dataFile.seek(i2*HashTree.BLOCK_SIZE)
 
			binData=dataFile.read(HashTree.BLOCK_SIZE)
 
 
			print("block #{0}: {1}...{2}".format(i2,binData[:5],binData[-5:]))
 
 
			outcoming.put((jsonData,binData),timeout=2)
 
			outcoming.writeMsg(jsonData,binData)
 
			i1=i2
 
 
	with Connection() as (incoming,outcoming):
 
		jsonData={"command":"end"}
 
		outcoming.put((jsonData,b""),timeout=2)
 
		outcoming.writeMsg({"command":"end"})
 
 
	print("closing...")
 
	dataFile.close()
 
 
 
if __name__=="__main__":
 
	blocksToTransfer=negotiate()
 
	sendData(blocksToTransfer)
 
 
	sys.exit(0)
src/networkers.py
➞
Show inline comments
 
import threading
 
import queue
 
import json
 
import json
 
 
 
class NetworkReader(threading.Thread):
 
class NetworkReader:
 
	def __init__(self,stream):
 
		threading.Thread.__init__(self,daemon=True)
 
		self.parent=threading.current_thread()
 
 
		self.stream=stream
 
		self.output=queue.Queue()
 
 
	def run(self):
 
		if not self.parent.is_alive(): return
 
		self.output.put(self.readMsg(),timeout=5)
 
 
	def readMsg(self):
 
		data=self.stream.readline()
 
		if not data: pass # !! raise something
 
		jsonLength=int(data.split(b":")[1].strip()) # "json-length: length" -> length
 
		data=self.stream.readline()
 
		if not data: pass # !! raise something
 
		binLength=int(data.split(b":")[1].strip()) # "bin-length: length" -> length
 
		jsonData=self.stream.read(jsonLength)
 
		jsonData=json.loads(str(jsonData,encoding="utf-8"))
 
		binData=self.stream.read(binLength)
 
 
		return (jsonData,binData)
 
 
 
class NetworkWriter(threading.Thread):
 
class NetworkWriter:
 
	def __init__(self,stream):
 
		threading.Thread.__init__(self,daemon=True)
 
		self.parent=threading.current_thread()
 
 
		self.stream=stream
 
		self.input=queue.Queue()
 
 
	def run(self):
 
		if not self.parent.is_alive(): return
 
		msg=self.input.get(timeout=5)
 
		if msg is None: return
 
		self.stream.write(self.prepMsg(msg))
 
	def writeMsg(self,*args):
 
		self.stream.write(self.prepMsg(*args))
 
		self.stream.flush()
 
 
	def prepMsg(self, msg):
 
		jsonData,binData=msg
 
	def prepMsg(self,jsonData,binData=b""):
 
		jsonData=bytes(json.dumps(jsonData)+"\n",encoding="utf-8")
 
		jsonLength=bytes("json-length: "+str(len(jsonData))+"\n",encoding="utf-8")
 
		binLength=bytes("bin-length: "+str(len(binData))+"\n",encoding="utf-8")
 
		return b"".join((jsonLength,binLength,jsonData,binData))
src/server.py
➞
Show inline comments
 
@@ -11,68 +11,63 @@ import shutil
 
origFilename=sys.argv[1]
 
filename=origFilename+"_"
 
shutil.copyfile(origFilename,filename)
 
 
 
class Connection:
 
	def __init__(self,server_socket):
 
		self.socket, address = server_socket.accept()
 
		print('Connected by', address)
 
		fr=self.socket.makefile(mode="rb")
 
		fw=self.socket.makefile(mode="wb")
 
 
		networkReader=NetworkReader(fr)
 
		networkReader.start()
 
		networkWriter=NetworkWriter(fw)
 
		networkWriter.start()
 
 
		self.incoming=networkReader.output # synchronized message queue
 
		self.outcoming=networkWriter.input
 
		self.incoming=NetworkReader(fr)
 
		self.outcoming=NetworkWriter(fw)
 
 
	def __enter__(self):
 
		return self.incoming,self.outcoming
 
 
	def __exit__(self, exc_type, exc_val, exc_tb):
 
		self.socket.close()
 
 
 
localTree=HashTree.fromFile(open(filename,mode="rb"))
 
 
ss = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
 
ss.bind(("",conf.port))
 
ss.listen(1)
 
 
blocksToTransfer=[]
 
nodeStack=collections.deque([0])
 
 
i1=-1
 
 
 
while True:
 
	with Connection(ss) as (incoming,outcoming):
 
		jsonData,binData=incoming.get(timeout=2)
 
		jsonData,binData=incoming.readMsg()
 
		dataFile=open(filename,mode="rb+")
 
 
		if jsonData["command"]=="init":
 
			assert jsonData["blockSize"]==localTree.BLOCK_SIZE
 
			assert jsonData["blockCount"]==localTree.leafCount
 
 
		elif jsonData["command"]=="req": # !! index out of range
 
			print("received request for node #{0}".format(jsonData["index"]))
 
			nodeHash=localTree.store[jsonData["index"]]
 
 
			jsonResponse={"command":"send", "index":jsonData["index"], "dataType":"hash"}
 
			binResponse=nodeHash
 
 
			outcoming.put((jsonResponse,binResponse),timeout=2)
 
			outcoming.writeMsg(jsonResponse,binResponse)
 
 
		elif jsonData["command"]=="send" and jsonData["dataType"]=="data": # needlessly allow hashes and data in mixed order
 
			print("received data block #{0}: {1}...{2}".format(jsonData["index"],binData[:5],binData[-5:]))
 
 
			i2=jsonData["index"]
 
			if i1+1!=i2:
 
				dataFile.seek(i2*localTree.BLOCK_SIZE)
 
			dataFile.write(binData)
 
			i1=i2
 
 
			# never update the hash tree
 
0 comments (0 inline, 0 general)
Server instance: Sirius-560 This site is powered by Kallithea 0.7.0, which is © 2010–2023 by various authors & licensed under GPLv3. – Support