diff --git a/languedoc.py b/languedoc.py --- a/languedoc.py +++ b/languedoc.py @@ -3,11 +3,12 @@ import re import random import itertools +from shared import identify, extract_ngram_freqs, TOP_NGRAM_COUNT + random.seed(19181028) CROSSVALIDATION_SOURCE_COUNT = 5 TEST_LENS = [8, 16, 32, 64] -TOP_NGRAM_COUNT = 6000 def preprocess(text): @@ -15,22 +16,6 @@ def preprocess(text): return text.lower() -def extract_ngram_freqs(text, k): - n = len(text) - d = dict() - - for i in range(0, n-k+1): - key = text[i:i+k] - if key.isspace(): - continue - - d[key] = d.get(key, 0) + 1 - - count = sum(d.values()) - - return {key: val/count for (key, val) in d.items()} - - def merge_ngram_freqs(freqs): n = len(freqs) res = dict() @@ -141,12 +126,6 @@ def cross_validate(sample_sets): return score / max_score, (score, max_score) -def identify(text, models): - sample = Sample(text=text) - - return min(map(lambda m: (m.compare(sample), m.language), models))[1] - - DATA_DIR = os.path.join(os.path.dirname(__file__), "data") LANG_DIRS = sorted([x.path for x in os.scandir(DATA_DIR)])