Changeset - 5c80ca07f00c
[Not reviewed]
default
0 16 0
Laman - 5 years ago 2020-05-23 19:02:49

reformatted whitespace with more respect for PEP-8
4 files changed with 25 insertions and 3 deletions:
0 comments (0 inline, 0 general)
src/client.py
Show inline comments
 
@@ -27,49 +27,55 @@ class Connection(BaseConnection):
 

	
 
		self._socket=sslContext.wrap_socket(sock)
 

	
 
		try:
 
			self._socket.connect((host,port))
 
		except (ConnectionRefusedError,OSError) as e:
 
			log.exception(e)
 
			print("Couldn't connect to {0}:{1}".format(host,port))
 
			raise FailedConnection()
 
		except ssl.SSLError as e:
 
			log.exception(e)
 
			print("Error creating SSL connection to {0}:{1}".format(host,port))
 
			raise FailedConnection()
 

	
 
		self.createNetworkers()
 
		print("Connected to {0}".format(host))
 

	
 

	
 
class Client(NetNode):
 
	def __init__(self,filename,treeFile=""):
 
		print(datetime.now(), "initializing...")
 
		super().__init__(filename,treeFile)
 

	
 
	def init(self,action):
 
		jsonData={"command":"init", "blockSize":self._tree.BLOCK_SIZE, "blockCount":self._tree.leafCount, "version":conf.version, "action":action}
 
		jsonData = {
 
			"command": "init",
 
			"blockSize": self._tree.BLOCK_SIZE,
 
			"blockCount": self._tree.leafCount,
 
			"version": conf.version,
 
			"action": action
 
		}
 
		self._outcoming.writeMsg(jsonData)
 
		jsonData,binData=self._incoming.readMsg()
 
		if jsonData["command"]=="deny":
 
			if jsonData["status"]==status.incompatible.version:
 
				raise DeniedConnection("Incompatible client version. Consider upgrading it.")
 
			raise DeniedConnection()
 
		assert jsonData["command"]=="init"
 
		if jsonData["version"]<conf.lowestCompatible:
 
			raise IncompatibleException("Incompatible server version. Consider upgrading it.")
 

	
 
	## Asks server for node hashes to determine which are to be transferred.
 
	#
 
	# Uses a binary HashTree, where item at k is hash of items at 2k+1, 2k+2.
 
	#
 
	# Requests nodes in order of a batch DFS. Needs stack of size O(treeDepth*batchSize). Nodes in each tree level are accessed in order.
 
	def negotiate(self):
 
		localTree=self._tree
 
		blocksToTransfer=[]
 
		nodeStack=collections.deque([0]) # root
 

	
 
		# determine which blocks to send
 
		print(datetime.now(), "negotiating:")
 
		progress=Progress(localTree.leafCount)
 
		while len(nodeStack)>0:
src/morevna.py
Show inline comments
 
@@ -8,96 +8,98 @@ import config as conf
 
import stats
 
from hashtree import HashTree
 
from client import Client, Connection as ClientConnection, FailedConnection, DeniedConnection, IncompatibleException
 
from server import Miniserver
 

	
 

	
 
def _checkFile(f):
 
	if not os.path.isfile(f):
 
		print("invalid file specified:",f,file=sys.stderr)
 
		sys.exit(1)
 

	
 

	
 
def buildTree(args):
 
	_checkFile(args.datafile)
 
	if os.path.isfile(args.treefile):
 
		treeMod=os.stat(args.treefile).st_mtime
 
		dataMod=os.stat(args.datafile).st_mtime
 
		if dataMod<treeMod and not args.force:
 
			print("tree file is up to date")
 
			return
 

	
 
	tree=HashTree.fromFile(args.datafile)
 
	tree.save(args.treefile)
 

	
 

	
 
def push(args):
 
	_checkFile(args.datafile)
 
	if args.tree:
 
		_checkFile(args.tree)
 
	if args.host: conf.hosts=[args.host]
 
	if args.port: conf.port=args.port
 

	
 
	c=Client(args.datafile,args.tree)
 
	for h in conf.hosts:
 
		host=splitHost(h,conf.port)
 
		stats.reset()
 
		try:
 
			with ClientConnection(*host) as con:
 
				c.setConnection(con)
 
				c.init("push")
 
				blocksToTransfer=c.negotiate()
 
				c.sendData(blocksToTransfer)
 
			print()
 
			print(stats.report())
 
			print()
 
		except FailedConnection: pass
 
		except DeniedConnection as e:
 
			print("Server {0}:{1} denied connection.".format(*host))
 
			print(e)
 
		except IncompatibleException as e: print(e)
 

	
 
def pull(args):
 
	_checkFile(args.datafile)
 
	if args.tree:
 
		_checkFile(args.tree)
 
	if args.host: conf.hosts=[args.host]
 
	if args.port: conf.port=args.port
 

	
 
	c=Client(args.datafile,args.tree)
 
	host=splitHost(conf.hosts[0],conf.port)
 
	try:
 
		with ClientConnection(*host) as con:
 
			c.setConnection(con)
 
			c.init("pull")
 
			blocksToTransfer=c.negotiate()
 
			c.pullData(blocksToTransfer,args.force)
 
		print()
 
		print(stats.report())
 
	except FailedConnection: pass
 
	except DeniedConnection as e:
 
		print("Server {0}:{1} denied connection.".format(*host))
 
		print(e)
 

	
 

	
 
def serve(args):
 
	_checkFile(args.datafile)
 
	if args.tree:
 
		_checkFile(args.tree)
 
	if args.host: conf.hosts.insert(0,args.host)
 
	if args.port: conf.port=args.port
 

	
 
	try:
 
		s=Miniserver(args.datafile,args.tree)
 
		spawnDaemon(s.serve)
 
	except Exception as e:
 
		log.exception("exception: %s",e)
 
		print("Failed to start:\n  ",e)
 

	
 

	
 
parser=ArgumentParser()
 
subparsers=parser.add_subparsers()
 

	
 
pBuild=subparsers.add_parser("build")
 
pBuild.add_argument("-f","--force",action="store_true",help="force tree rebuild")
 
pBuild.add_argument("treefile", help="stored hash tree location")
 
pBuild.add_argument("datafile")
 
pBuild.set_defaults(func=buildTree)
 

	
src/morevna.sh
Show inline comments
 
#!/bin/bash
 

	
 
# setup encrypted container
 
#sudo losetup -f ext2.img
 
#sudo cryptsetup luksFormat /dev/loop0
 
#sudo cryptsetup open --type=luks /dev/loop0 ext2luks
 
#sudo mkfs.ext2 /dev/mapper/ext2luks
 

	
 
# generate certificate
 
# openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -days 365 -nodes -subj "/CN=name"
 

	
 
set -e
 
DIRNAME=`dirname $0`
 

	
 
ssh-add
 

	
 
rsync -av 19x19.cz:letsencrypt.tar.xz.gpg ~/Projekty/
 
rsync -av 19x19.cz:/home/laman/projekty/laman/db_backup/ ~/Programy/db_backup/
 
rsync -av 19x19.cz:/home/laman/projekty/gkul/db_backup/ ~/Programy/db_backup/
 
find ~/Programy/db_backup/ -mtime +30 -type f -delete
 

	
 
rdiff-backup -v 3 19x19.cz::/home/laman/projekty/laman/static/media ~/Programy/static_backup/laman
 
rdiff-backup -v 3 19x19.cz::/home/laman/projekty/gkul/static/media ~/Programy/static_backup/gkul
 
rsync -av 19x19.cz:/home/laman/projekty/copobot/logs ~/Programy/static_backup/copobot
 

	
 
sudo losetup -f ~/ext2.img
 
sudo cryptsetup open --type=luks /dev/loop0 ext2luks
 
sudo mount /dev/mapper/ext2luks ~/temp
 

	
 
sudo rdiff-backup -v 5 ~/Dokumenty ~/temp/Dokumenty
 
sudo rdiff-backup -v 5 --exclude-regexp '/__pycache__/' ~/Projekty ~/temp/Projekty
 
sudo rdiff-backup -v 5 --exclude '**/__pycache__' ~/Projekty ~/temp/Projekty
 
sudo rdiff-backup -v 5 ~/Obrázky ~/temp/Obrázky
 

	
 
sudo umount /dev/mapper/ext2luks
 
sudo cryptsetup close ext2luks
 
sudo losetup -d /dev/loop0
 

	
 
echo
 

	
 
python $DIRNAME/morevna.py build ~/ext2.bin ~/ext2.img
 
python $DIRNAME/morevna.py push --tree ~/ext2.bin ~/ext2.img
src/stats.py
Show inline comments
 
@@ -12,37 +12,44 @@ def logReceived(data):
 
	stats.received+=len(data)
 

	
 

	
 
def logSent(data):
 
	stats.sent+=len(data)
 

	
 

	
 
def logExchangedNode(k=1):
 
	stats.exchangedNodes+=k
 

	
 

	
 
def logTransferredBlock(k=1):
 
	stats.transferredBlocks+=k
 

	
 

	
 
def reset():
 
	global stats
 
	stats=Stats()
 

	
 

	
 
def report():
 
	return """received {rf} ({r:,} B)
 
sent {sf} ({s:,} B)
 
exchanged {nodes:,} hash tree nodes
 
transferred {blocks:,} blocks""".format(rf=formatBytes(stats.received), r=stats.received, sf=formatBytes(stats.sent), s=stats.sent, nodes=stats.exchangedNodes, blocks=stats.transferredBlocks)
 
transferred {blocks:,} blocks""".format(
 
		rf=formatBytes(stats.received),
 
		r=stats.received,
 
		sf=formatBytes(stats.sent),
 
		s=stats.sent,
 
		nodes=stats.exchangedNodes,
 
		blocks=stats.transferredBlocks
 
	)
 

	
 

	
 
def formatBytes(x):
 
	exts=["B","kiB","MiB","GiB","TiB","PiB"]
 
	i=0
 
	while x>1024:
 
		x/=1024
 
		i+=1
 
	if x>=100: x=round(x)
 
	elif x>=10: x=round(x,1)
 
	else: x=round(x,2)
 
	return "{0} {1}".format(x,exts[i])
0 comments (0 inline, 0 general)