Languedoc Changeset - 1cae4ecc8978

Changeset - 1cae4ecc8978

Parent rev.

Child rev.

[Not reviewed]

default

0 1 0

Laman - 3 years ago 2022-09-18 16:05:43

created a Sample object

1 file changed with 49 insertions and 7 deletions:

languedoc.py

0 comments (0 inline, 0 general)

languedoc.py

➞

Show inline comments

 import os
 import re
 import random
 random.seed(19181028)
 def preprocess(text):
-	text = re.sub(r"[\W\d]+", " ", text)
+	text = re.sub(r"[\W\d_]+", " ", " "+text+" ")
 	return text.lower()
 def extract_ngram_freqs(text, k):
 	n = len(text)
 	d = dict()
 	for i in range(0, n-k):
 		key = text[i:i+k]
 		if key.isspace():
 			continue
 		d[key] = d.get(key, 0) + 1
 	count = sum(d.values())
 	return {key: val/count for (key, val) in d.items()}
 def merge_ngram_freqs(freqs):
 	n = len(freqs)
 	res = dict()
 	for d in freqs:
 		for (key, val) in d.items():
 			res.setdefault(key, 0)
 			res[key] += val/n
 	return res
 class Sample:
 	def __init__(self, language="??", text=""):
 		self.language = language
 		self.frequencies = [dict(), dict(), dict()]
 		if text:
 			self._extract(text)
 	def _extract(self, text):
 		for k in range(1, 4):
 			self.frequencies[k-1] = extract_ngram_freqs(text, k)
 	@staticmethod
 	def merge(samples):
 		assert len({x.language for x in samples}) == 1
 		res = Sample(samples[0].language)
 		res.frequencies = []
 		for freqs in zip(*(x.frequencies for x in samples)):
 			res.frequencies.append(merge_ngram_freqs(freqs))
 		return res
 	def compare(self, other):
 		pass
 	def print_overview(self):
 		print(f"Sample({self.language}):")
 		for freqs in self.frequencies:
 			x = [
 				(k, round(v, 3))
 				for (k, v) in sorted(freqs.items(), key=lambda kv: -kv[1])
+			]
 			print("  ", x[:8], "...", x[-8:])
 		print()
 DATA_DIR = os.path.join(os.path.dirname(__file__), "data")
 LANG_DIRS = [x.path for x in os.scandir(DATA_DIR)]
 models = dict()
 for d in LANG_DIRS:
 	models = [[], [], []]
 	lang = os.path.basename(d)
 	samples = []
 	for file in os.scandir(d):
 		with open(file) as f:
 			text = f.read()
 			text = preprocess(text)
 			print(f"{file.name} ({len(text)})")
 			print(text[:256])
 			print()
 			for k in range(1, 4):
 				models[k-1].append(extract_ngram_freqs(text, k))
 			samples.append(Sample(lang, text))
 			samples[-1].print_overview()
 	models = [merge_ngram_freqs(sources) for sources in models]
 	print(sorted(((key, round(val, 3)) for (key, val) in models[0].items()), key=lambda kv: -kv[1]))
 	print(sorted(((key, round(val, 3)) for (key, val) in models[1].items()), key=lambda kv: -kv[1]))
 	models[lang] = Sample.merge(samples)
 	models[lang].print_overview()

0 comments (0 inline, 0 general)