Languedoc Changeset - f6c340624f3f

Changeset - f6c340624f3f

Parent rev.

Child rev.

[Not reviewed]

default

0 1 0

Laman - 3 years ago 2022-09-30 21:18:01

optimized the prediction code

1 file changed with 12 insertions and 8 deletions:

languedoc.py

0 comments (0 inline, 0 general)

languedoc.py

➞

Show inline comments

@@ @@ -7,7 +7,7 @@ random.seed(19181028) @@
 CROSSVALIDATION_SOURCE_COUNT = 5
 TEST_LENS = [8, 16, 32, 64]
-TOP_NGRAM_COUNT = 4000
+TOP_NGRAM_COUNT = 6000
 def preprocess(text):
@@ @@ -47,6 +47,7 @@ class Sample: @@
 	def __init__(self, language="??", text=""):
 		self.language = language
 		self.frequencies = dict()
 		self._ranked_ngrams = dict()
 		if text:
 			self._extract(text)
@@ @@ -64,18 +65,21 @@ class Sample: @@
 		return res
 	@property
 	def ranked_ngrams(self):
 		if not self._ranked_ngrams:
 			ordered_ngrams = sorted(self.frequencies.items(), key=lambda kv: -kv[1])[:TOP_NGRAM_COUNT]
 			self._ranked_ngrams = dict(zip([key for (key, freq) in ordered_ngrams], itertools.count(0)))
 		return self._ranked_ngrams
 	def compare(self, other):
 		"""take k most common
 		use frequencies x order
 		use letter, digrams, trigrams
 		use absolute x square"""
 		ordered_own_ngrams = sorted(self.frequencies.items(), key=lambda kv: -kv[1])[:TOP_NGRAM_COUNT]
 		ordered_other_ngrams = sorted(other.frequencies.items(), key=lambda kv: -kv[1])[:TOP_NGRAM_COUNT]
 		ranked_own_ngrams = dict(zip([key for (key, freq) in ordered_own_ngrams], itertools.count(0)))
 		ranked_other_ngrams = dict(zip([key for (key, freq) in ordered_other_ngrams], itertools.count(0)))
 		res = sum(abs(v-ranked_other_ngrams.get(k, TOP_NGRAM_COUNT)) for (k, v) in ranked_own_ngrams.items()) + \
 					sum(abs(v-ranked_own_ngrams.get(k, TOP_NGRAM_COUNT)) for (k, v) in ranked_other_ngrams.items())
 		res = sum(abs(v-other.ranked_ngrams.get(k, len(other.ranked_ngrams))) for (k, v) in self.ranked_ngrams.items()) + \
 					sum(abs(v-self.ranked_ngrams.get(k, len(self.ranked_ngrams))) for (k, v) in other.ranked_ngrams.items())
 		return res

0 comments (0 inline, 0 general)