Languedoc Changeset - d443541818b2

Changeset - d443541818b2

Parent rev.

Child rev.

[Not reviewed]

default

2 0 3

Laman - 2 years ago 2022-10-12 21:03:29

changed the project layout

3 files changed with 5 insertions and 4 deletions:

src/languedoc/__init__.py

src/languedoc/predict.py

src/languedoc/train.py

0 comments (0 inline, 0 general)

src/languedoc/__init__.py

➞

Show inline comments

		new file 100644
		from .predict import identify

src/languedoc/predict.py

➞

Show inline comments

@@ file renamed from shared.py to src/languedoc/predict.py @@
 import os
 import re
 import itertools
 import json
 import gzip
 TOP_NGRAM_COUNT = 3000
 MODEL_PATH = os.path.join(os.path.dirname(__file__), "models.json.gz")
+MODEL_PATH = os.path.join(os.path.dirname(__file__), "../../models.json.gz")
 def preprocess(text):
 	text = re.sub(r"[\W\d_]+", " ", " "+text+" ")
 	return text.lower()
 def extract_kgram_freqs(text, k):
 	n = len(text)
 	d = dict()
 	for i in range(0, n-k+1):

src/languedoc/train.py

➞

Show inline comments

@@ file renamed from languedoc.py to src/languedoc/train.py @@
 import os
 import random
 import itertools
 import json
 import gzip
-from shared import preprocess, identify, extract_ngram_freqs, rank_ngram_freqs, Sample
+from predict import preprocess, identify, extract_ngram_freqs, rank_ngram_freqs, Sample
 random.seed(19181028)
 CROSSVALIDATION_SOURCE_COUNT = 5
 TEST_LENS = [8, 16, 32, 64]
 def merge_ngram_freqs(freqs):
 	n = len(freqs)
 	res = dict()
 	for d in freqs:
@@ @@ -63,27 +63,27 @@ def cross_validate(sample_sets): @@
 					j = random.randrange(0, len(test_text)-k)
 					t = test_text[j:j+k]
 					predicted_lang = identify(t, test_models)
 					if predicted_lang == real_lang:
 						score += 1
 					else:
 						print(real_lang, predicted_lang, t)
 					max_score += 1
 	return score / max_score, (score, max_score)
 DATA_DIR = os.path.join(os.path.dirname(__file__), "data")
+DATA_DIR = os.path.join(os.path.dirname(__file__), "../../data")
 LANG_DIRS = sorted([x.path for x in os.scandir(DATA_DIR)])
 MODEL_PATH = os.path.join(os.path.dirname(__file__), "models.json.gz")
+MODEL_PATH = os.path.join(os.path.dirname(__file__), "../../models.json.gz")
 if __name__ == "__main__":
 	samples = []
 	for d in LANG_DIRS:
 		lang = os.path.basename(d)
 		lang_samples = SampleSet(lang)
 		samples.append(lang_samples)
 		for file in sorted(os.scandir(d), key=lambda f: f.name):
 			with open(file) as f:
 				text = f.read()

0 comments (0 inline, 0 general)