Files
@ 13603ac261f1
Branch filter:
Location: Morevna/src/hashtree.py - annotation
13603ac261f1
2.6 KiB
text/x-python
minor enhancements
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 | bacd3b2d37aa bacd3b2d37aa 34f4027c1bd6 b052f27e1cbc 34f4027c1bd6 4b88aca70fbc 13d0327a4abb 34f4027c1bd6 8bb6a904d50b 8bb6a904d50b 8bb6a904d50b 8bb6a904d50b 34f4027c1bd6 34f4027c1bd6 34f4027c1bd6 34f4027c1bd6 34f4027c1bd6 34f4027c1bd6 e6752944fe6b 34f4027c1bd6 34f4027c1bd6 026618d6681b 34f4027c1bd6 34f4027c1bd6 d72018278450 d72018278450 d72018278450 d72018278450 d72018278450 d72018278450 b052f27e1cbc d72018278450 4b88aca70fbc d72018278450 d72018278450 8bb6a904d50b 13d0327a4abb 4b88aca70fbc 4b88aca70fbc 34f4027c1bd6 34f4027c1bd6 34f4027c1bd6 e6752944fe6b e6752944fe6b e6752944fe6b e6752944fe6b e6752944fe6b e6752944fe6b e6752944fe6b e6752944fe6b e6752944fe6b e6752944fe6b e6752944fe6b 5673cf6b1207 e6752944fe6b e6752944fe6b e6752944fe6b e6752944fe6b e6752944fe6b 34f4027c1bd6 34f4027c1bd6 026618d6681b 34f4027c1bd6 34f4027c1bd6 026618d6681b 026618d6681b 34f4027c1bd6 34f4027c1bd6 34f4027c1bd6 34f4027c1bd6 34f4027c1bd6 34f4027c1bd6 34f4027c1bd6 34f4027c1bd6 34f4027c1bd6 34f4027c1bd6 34f4027c1bd6 8bb6a904d50b 34f4027c1bd6 34f4027c1bd6 34f4027c1bd6 34f4027c1bd6 b052f27e1cbc 4b88aca70fbc 34f4027c1bd6 8bb6a904d50b 4b88aca70fbc 4b88aca70fbc bacd3b2d37aa bacd3b2d37aa bacd3b2d37aa bacd3b2d37aa bacd3b2d37aa bacd3b2d37aa bacd3b2d37aa bacd3b2d37aa bacd3b2d37aa bacd3b2d37aa bacd3b2d37aa bacd3b2d37aa bacd3b2d37aa bacd3b2d37aa bacd3b2d37aa bacd3b2d37aa | import collections
import hashlib
import os
from datetime import datetime
from util import Progress
def hashBlock(data):
return hashlib.sha256(data).digest()[-HashTree.HASH_LEN:]
class HashTree:
HASH_LEN=16 # bytes
BLOCK_SIZE=4096 # bytes
## Prepares a tree containing leafCount leaves.
def __init__(self,leafCount):
self.store=[b""]*(leafCount*2-1)
self.leafStart=leafCount-1
self.leafCount=leafCount
self._index=self.leafStart
@classmethod
def fromFile(cls,filename):
with open(filename,"rb") as f:
stat=os.fstat(f.fileno())
size=stat.st_size # !! symlinks
leafCount=(size-1)//HashTree.BLOCK_SIZE+1 # number of leaf blocks
res=cls(leafCount)
print(datetime.now(), "hashing file:")
progress=Progress(leafCount)
for i in range(leafCount):
data=f.read(HashTree.BLOCK_SIZE)
res.insertLeaf(hashBlock(data))
progress.p(i)
progress.done()
res.buildTree()
return res
@classmethod
def load(cls,filename):
with open(filename,"rb") as f:
stat=os.fstat(f.fileno())
size=stat.st_size
nodeCount=size//HashTree.HASH_LEN
res=cls((nodeCount+1)//2)
for i in range(nodeCount):
res.store[i]=f.read(HashTree.HASH_LEN)
return res
def save(self,filename):
with open(filename,"wb") as f:
for h in self.store:
f.write(h)
## Inserts a leaf at the first empty position.
#
# Useful and used only during the tree construction.
def insertLeaf(self,h):
self.store[self._index]=h
self._index+=1
## Updates a hash stored in the leaf.
def updateLeaf(self,index,h):
if index<self.leafStart: raise IndexError()
self.store[index]=h
self.updateNode((index-1)//2)
## Updates the node at index and all its ancestors.
def updateNode(self,index):
while index>=0:
self.store[index]=hashBlock(self.store[index*2+1]+self.store[index*2+2])
index=(index-1)//2
## Fast construction of the tree over the leaves. O(n).
def buildTree(self):
print(datetime.now(), "building tree:")
progress=Progress(-1, self.leafStart-1)
for i in range(self.leafStart-1,-1,-1):
self.store[i]=hashBlock(self.store[i*2+1]+self.store[i*2+2])
progress.p(i)
progress.done()
## Update faster than repeated insertLeaf.
def batchUpdate(self,keysHashes):
queue=collections.deque()
for (k,v) in sorted(keysHashes):
self.store[k]=v
parentK=(k-1)//2
if len(queue)==0 or queue[-1]!=parentK:
queue.append(parentK)
while len(queue)>0:
k=queue.pop()
self.store[k]=hashBlock(self.store[k*2+1]+self.store[k*2+2])
parentK=(k-1)//2
if (len(queue)==0 or queue[0]!=parentK) and k!=0:
queue.appendleft(parentK)
|