diff --git a/shared.py b/shared.py --- a/shared.py +++ b/shared.py @@ -1,7 +1,7 @@ import re import itertools -TOP_NGRAM_COUNT = 5000 +TOP_NGRAM_COUNT = 3000 def preprocess(text): @@ -69,9 +69,12 @@ class Sample: use frequencies x order use letter, digrams, trigrams use absolute x square""" - """make a set difference of keys, multiply its size by the max score""" - res = sum(abs(v-other.ranked_ngrams.get(k, len(other.ranked_ngrams))) for (k, v) in self.ranked_ngrams.items()) + \ - sum(abs(v-self.ranked_ngrams.get(k, len(self.ranked_ngrams))) for (k, v) in other.ranked_ngrams.items()) + m = len(other.ranked_ngrams) + + res = sum( + (abs(v - other.ranked_ngrams[k]) if k in other.ranked_ngrams else m) + for (k, v) in self.ranked_ngrams.items() + ) return res @@ -79,4 +82,4 @@ class Sample: def identify(text, models): sample = Sample.extract(text) - return sorted(models, key=lambda m: m.compare(sample))[0].language + return sorted(models, key=lambda m: sample.compare(m))[0].language