# HG changeset patch
# User Laman
# Date 2022-09-18 16:05:43
# Node ID 1cae4ecc8978965c7fd461a35a680ba459fb6c9b
# Parent  1c7a7c3926e60935690497097c386a9d9be6e842

created a Sample object

diff --git a/languedoc.py b/languedoc.py
--- a/languedoc.py
+++ b/languedoc.py
@@ -6,7 +6,7 @@ random.seed(19181028)
 
 
 def preprocess(text):
-	text = re.sub(r"[\W\d]+", " ", text)
+	text = re.sub(r"[\W\d_]+", " ", " "+text+" ")
 	return text.lower()
 
 
@@ -38,11 +38,54 @@ def merge_ngram_freqs(freqs):
 	return res
 
 
+class Sample:
+	def __init__(self, language="??", text=""):
+		self.language = language
+		self.frequencies = [dict(), dict(), dict()]
+
+		if text:
+			self._extract(text)
+
+	def _extract(self, text):
+		for k in range(1, 4):
+			self.frequencies[k-1] = extract_ngram_freqs(text, k)
+
+	@staticmethod
+	def merge(samples):
+		assert len({x.language for x in samples}) == 1
+
+		res = Sample(samples[0].language)
+		res.frequencies = []
+
+		for freqs in zip(*(x.frequencies for x in samples)):
+			res.frequencies.append(merge_ngram_freqs(freqs))
+
+		return res
+
+	def compare(self, other):
+		pass
+
+	def print_overview(self):
+		print(f"Sample({self.language}):")
+
+		for freqs in self.frequencies:
+			x = [
+				(k, round(v, 3))
+				for (k, v) in sorted(freqs.items(), key=lambda kv: -kv[1])
+			]
+			print("  ", x[:8], "...", x[-8:])
+
+		print()
+
+
 DATA_DIR = os.path.join(os.path.dirname(__file__), "data")
 LANG_DIRS = [x.path for x in os.scandir(DATA_DIR)]
 
+models = dict()
+
 for d in LANG_DIRS:
-	models = [[], [], []]
+	lang = os.path.basename(d)
+	samples = []
 
 	for file in os.scandir(d):
 		with open(file) as f:
@@ -52,9 +95,8 @@ for d in LANG_DIRS:
 			print(text[:256])
 			print()
 
-			for k in range(1, 4):
-				models[k-1].append(extract_ngram_freqs(text, k))
+			samples.append(Sample(lang, text))
+			samples[-1].print_overview()
 
-	models = [merge_ngram_freqs(sources) for sources in models]
-	print(sorted(((key, round(val, 3)) for (key, val) in models[0].items()), key=lambda kv: -kv[1]))
-	print(sorted(((key, round(val, 3)) for (key, val) in models[1].items()), key=lambda kv: -kv[1]))
+	models[lang] = Sample.merge(samples)
+	models[lang].print_overview()