diff --git a/shared.py b/shared.py --- a/shared.py +++ b/shared.py @@ -1,9 +1,15 @@ +import re import itertools TOP_NGRAM_COUNT = 5000 -def extract_ngram_freqs(text, k): +def preprocess(text): + text = re.sub(r"[\W\d_]+", " ", " "+text+" ") + return text.lower() + + +def extract_kgram_freqs(text, k): n = len(text) d = dict() @@ -19,26 +25,44 @@ def extract_ngram_freqs(text, k): return {key: val/count for (key, val) in d.items()} -class Sample: - def __init__(self, language="??", text=""): - self.language = language - self.frequencies = dict() - self._ranked_ngrams = dict() +def extract_ngram_freqs(text): + frequencies = {} + + for k in range(1, 4): + frequencies.update(extract_kgram_freqs(text, k)) + + return frequencies + - if text: - self._extract(text) +def rank_ngram_freqs(frequencies): + ordered_ngrams = sorted(frequencies.items(), key=lambda kv: -kv[1])[:TOP_NGRAM_COUNT] + return dict(zip([key for (key, freq) in ordered_ngrams], itertools.count(0))) + + +def extract_ranked_ngrams(text): + frequencies = extract_ngram_freqs(text) + return rank_ngram_freqs(frequencies) + - def _extract(self, text): - for k in range(1, 4): - self.frequencies.update(extract_ngram_freqs(text, k)) +class Sample: + def __init__(self, language, ranked_ngrams): + self.language = language + self.ranked_ngrams = ranked_ngrams + + @classmethod + def extract(cls, text, language="??"): + return cls(language, extract_ranked_ngrams(preprocess(text))) - @property - def ranked_ngrams(self): - if not self._ranked_ngrams: - ordered_ngrams = sorted(self.frequencies.items(), key=lambda kv: -kv[1])[:TOP_NGRAM_COUNT] - self._ranked_ngrams = dict(zip([key for (key, freq) in ordered_ngrams], itertools.count(0))) + @classmethod + def load(cls, exported): + ranked_ngrams = {key: order for (order, key) in enumerate(exported["ngrams"])} + return cls(exported["language"], ranked_ngrams) - return self._ranked_ngrams + def export(self): + return { + "language": self.language, + "ngrams": [key for (key, order) in sorted(self.ranked_ngrams.items(), key=lambda key_order: key_order[1])] + } def compare(self, other): """take k most common @@ -51,20 +75,8 @@ class Sample: return res - def print_overview(self): - print(f"Sample({self.language}):") - - for freqs in self.frequencies: - x = [ - (k, round(v, 3)) - for (k, v) in sorted(freqs.items(), key=lambda kv: -kv[1]) - ] - print(" ", x[:8], "...", x[-8:]) - - print() - def identify(text, models): - sample = Sample(text=text) + sample = Sample.extract(text) return sorted(models, key=lambda m: m.compare(sample))[0].language