import re
import itertools
TOP_NGRAM_COUNT = 5000
TOP_NGRAM_COUNT = 3000
def preprocess(text):
text = re.sub(r"[\W\d_]+", " ", " "+text+" ")
return text.lower()
def extract_kgram_freqs(text, k):
n = len(text)
d = dict()
for i in range(0, n-k+1):
@@ -60,23 +60,26 @@ class Sample:
def export(self):
return {
"language": self.language,
"ngrams": [key for (key, order) in sorted(self.ranked_ngrams.items(), key=lambda key_order: key_order[1])]
}
def compare(self, other):
"""take k most common
use frequencies x order
use letter, digrams, trigrams
use absolute x square"""
"""make a set difference of keys, multiply its size by the max score"""
res = sum(abs(v-other.ranked_ngrams.get(k, len(other.ranked_ngrams))) for (k, v) in self.ranked_ngrams.items()) + \
sum(abs(v-self.ranked_ngrams.get(k, len(self.ranked_ngrams))) for (k, v) in other.ranked_ngrams.items())
m = len(other.ranked_ngrams)
res = sum(
(abs(v - other.ranked_ngrams[k]) if k in other.ranked_ngrams else m)
for (k, v) in self.ranked_ngrams.items()
)
return res
def identify(text, models):
sample = Sample.extract(text)
return sorted(models, key=lambda m: m.compare(sample))[0].language
return sorted(models, key=lambda m: sample.compare(m))[0].language
Status change: