diff --git a/languedoc.py b/languedoc.py --- a/languedoc.py +++ b/languedoc.py @@ -7,7 +7,7 @@ random.seed(19181028) CROSSVALIDATION_SOURCE_COUNT = 5 TEST_LENS = [8, 16, 32, 64] -TOP_NGRAM_COUNT = 4000 +TOP_NGRAM_COUNT = 6000 def preprocess(text): @@ -47,6 +47,7 @@ class Sample: def __init__(self, language="??", text=""): self.language = language self.frequencies = dict() + self._ranked_ngrams = dict() if text: self._extract(text) @@ -64,18 +65,21 @@ class Sample: return res + @property + def ranked_ngrams(self): + if not self._ranked_ngrams: + ordered_ngrams = sorted(self.frequencies.items(), key=lambda kv: -kv[1])[:TOP_NGRAM_COUNT] + self._ranked_ngrams = dict(zip([key for (key, freq) in ordered_ngrams], itertools.count(0))) + + return self._ranked_ngrams + def compare(self, other): """take k most common use frequencies x order use letter, digrams, trigrams use absolute x square""" - ordered_own_ngrams = sorted(self.frequencies.items(), key=lambda kv: -kv[1])[:TOP_NGRAM_COUNT] - ordered_other_ngrams = sorted(other.frequencies.items(), key=lambda kv: -kv[1])[:TOP_NGRAM_COUNT] - ranked_own_ngrams = dict(zip([key for (key, freq) in ordered_own_ngrams], itertools.count(0))) - ranked_other_ngrams = dict(zip([key for (key, freq) in ordered_other_ngrams], itertools.count(0))) - - res = sum(abs(v-ranked_other_ngrams.get(k, TOP_NGRAM_COUNT)) for (k, v) in ranked_own_ngrams.items()) + \ - sum(abs(v-ranked_own_ngrams.get(k, TOP_NGRAM_COUNT)) for (k, v) in ranked_other_ngrams.items()) + res = sum(abs(v-other.ranked_ngrams.get(k, len(other.ranked_ngrams))) for (k, v) in self.ranked_ngrams.items()) + \ + sum(abs(v-self.ranked_ngrams.get(k, len(self.ranked_ngrams))) for (k, v) in other.ranked_ngrams.items()) return res