import os
import re
import random

random.seed(19181028)


def preprocess(text):
	text = re.sub(r"[\W\d_]+", " ", " "+text+" ")
	return text.lower()


def extract_ngram_freqs(text, k):
	n = len(text)
	d = dict()

	for i in range(0, n-k):
		key = text[i:i+k]
		if key.isspace():
			continue

		d[key] = d.get(key, 0) + 1

	count = sum(d.values())

	return {key: val/count for (key, val) in d.items()}


def merge_ngram_freqs(freqs):
	n = len(freqs)
	res = dict()

	for d in freqs:
		for (key, val) in d.items():
			res.setdefault(key, 0)
			res[key] += val/n

	return res


class Sample:
	def __init__(self, language="??", text=""):
		self.language = language
		self.frequencies = [dict(), dict(), dict()]

		if text:
			self._extract(text)

	def _extract(self, text):
		for k in range(1, 4):
			self.frequencies[k-1] = extract_ngram_freqs(text, k)

	@staticmethod
	def merge(samples):
		assert len({x.language for x in samples}) == 1

		res = Sample(samples[0].language)
		res.frequencies = []

		for freqs in zip(*(x.frequencies for x in samples)):
			res.frequencies.append(merge_ngram_freqs(freqs))

		return res

	def compare(self, other):
		pass

	def print_overview(self):
		print(f"Sample({self.language}):")

		for freqs in self.frequencies:
			x = [
				(k, round(v, 3))
				for (k, v) in sorted(freqs.items(), key=lambda kv: -kv[1])
			]
			print("  ", x[:8], "...", x[-8:])

		print()


DATA_DIR = os.path.join(os.path.dirname(__file__), "data")
LANG_DIRS = [x.path for x in os.scandir(DATA_DIR)]

models = dict()

for d in LANG_DIRS:
	lang = os.path.basename(d)
	samples = []

	for file in os.scandir(d):
		with open(file) as f:
			text = f.read()
			text = preprocess(text)
			print(f"{file.name} ({len(text)})")
			print(text[:256])
			print()

			samples.append(Sample(lang, text))
			samples[-1].print_overview()

	models[lang] = Sample.merge(samples)
	models[lang].print_overview()