Changeset - a27c2661a846
[Not reviewed]
default
0 1 0
Laman - 2 years ago 2022-10-10 22:12:27

fixed the prediction code to match the original paper
1 file changed with 8 insertions and 5 deletions:
0 comments (0 inline, 0 general)
shared.py
Show inline comments
 
import re
 
import itertools
 

	
 
TOP_NGRAM_COUNT = 5000
 
TOP_NGRAM_COUNT = 3000
 

	
 

	
 
def preprocess(text):
 
@@ -69,9 +69,12 @@ class Sample:
 
		use frequencies x order
 
		use letter, digrams, trigrams
 
		use absolute x square"""
 
		"""make a set difference of keys, multiply its size by the max score"""
 
		res = sum(abs(v-other.ranked_ngrams.get(k, len(other.ranked_ngrams))) for (k, v) in self.ranked_ngrams.items()) + \
 
					sum(abs(v-self.ranked_ngrams.get(k, len(self.ranked_ngrams))) for (k, v) in other.ranked_ngrams.items())
 
		m = len(other.ranked_ngrams)
 

	
 
		res = sum(
 
			(abs(v - other.ranked_ngrams[k]) if k in other.ranked_ngrams else m)
 
			for (k, v) in self.ranked_ngrams.items()
 
		)
 

	
 
		return res
 

	
 
@@ -79,4 +82,4 @@ class Sample:
 
def identify(text, models):
 
	sample = Sample.extract(text)
 

	
 
	return sorted(models, key=lambda m: m.compare(sample))[0].language
 
	return sorted(models, key=lambda m: sample.compare(m))[0].language
0 comments (0 inline, 0 general)