Languedoc Changeset - ba1303bfd58c

Changeset - ba1303bfd58c

Parent rev.

Child rev.

[Not reviewed]

default

0 1 1

Laman - 23 months ago 2023-05-06 17:19:39

determinized the model generation

2 files changed with 3 insertions and 2 deletions:

src/languedoc/models.json.gz

bin+new file

src/languedoc/train.py

0 comments (0 inline, 0 general)

src/languedoc/models.json.gz

➞

Show inline comments

		new file 100644
		binary diff not shown

src/languedoc/train.py

➞

Show inline comments

@@ @@ -110,14 +110,15 @@ def train(data_dir: str, model_path: str @@
 				text = f.read()
 				text = preprocess(text)
 				print(f"{lang}: {file.name} ({len(text)})")
 				lang_samples.add(text)
 	with gzip.open(model_path, mode="wt", encoding="utf-8") as f:
 		json.dump([sample_set.create_model().export() for sample_set in samples], f, ensure_ascii=False)
 	with gzip.GzipFile(model_path, mode="wb", mtime=0) as f:
 		s = json.dumps([sample_set.create_model().export() for sample_set in samples], ensure_ascii=False, sort_keys=True)
 		f.write(s.encode("utf-8"))
 	print(cross_validate(samples))
 DATA_DIR = os.path.join(os.path.dirname(__file__), "../../data")
 MODEL_PATH = os.path.join(os.path.dirname(__file__), "models.json.gz")

0 comments (0 inline, 0 general)