Changeset - f8fe5a65e7fc
Parent rev.
Child rev.
[Not reviewed]
default
0 2 0
Laman - 2 years ago 2022-10-11 22:33:33

model saving and loading
2 files changed with 19 insertions and 5 deletions:
languedoc.py
6
shared.py
13
5
0 comments (0 inline, 0 general)
languedoc.py
➞
Show inline comments
 
import os
 
import random
 
import itertools
 
import json
 
import gzip
 

			




	
	
	
		
 
from shared import preprocess, identify, extract_ngram_freqs, rank_ngram_freqs, Sample

			




	
	
	
		
 

			




	
	
	
		
 
random.seed(19181028)

			




	
	
	
		
 

			




	
	
	
		
 
CROSSVALIDATION_SOURCE_COUNT = 5

			




	
	
		
 
@@ -69,12 +71,13 @@ def cross_validate(sample_sets):

			




	
	
	
		
 

			




	
	
	
		
 
	return score / max_score, (score, max_score)

			




	
	
	
		
 

			




	
	
	
		
 

			




	
	
	
		
 
DATA_DIR = os.path.join(os.path.dirname(__file__), "data")

			




	
	
	
		
 
LANG_DIRS = sorted([x.path for x in os.scandir(DATA_DIR)])

			




	
	
	
		
 
MODEL_PATH = os.path.join(os.path.dirname(__file__), "models.json.gz")

			




	
	
	
		
 

			




	
	
	
		
 
if __name__ == "__main__":

			




	
	
	
		
 
	samples = []

			




	
	
	
		
 

			




	
	
	
		
 
	for d in LANG_DIRS:

			




	
	
	
		
 
		lang = os.path.basename(d)

			




	
	
		
 
@@ -86,7 +89,10 @@ if __name__ == "__main__":

			




	
	
	
		
 
				text = f.read()

			




	
	
	
		
 
				text = preprocess(text)

			




	
	
	
		
 
				print(f"{lang}: {file.name} ({len(text)})")

			




	
	
	
		
 

			




	
	
	
		
 
				lang_samples.add(text)

			




	
	
	
		
 

			




	
	
	
		
 
	with gzip.open(MODEL_PATH, mode="wt", encoding="utf-8") as f:

			




	
	
	
		
 
		json.dump([sample_set.create_model().export() for sample_set in samples], f, ensure_ascii=False)

			




	
	
	
		
 

			




	
	
	
		
 
	print(cross_validate(samples))

			




        

    


    
    

    

        

                

                    shared.py
                

                

                  
                      
                        

                      ➞

                      
                        
                  

                  
                      
                  
                      
                  
                      
                  
                      
                  
                  
                

                

                    Show inline comments
                    
                

        

        

            



	
	
	
		
 
import os

			




	
	
	
		
 
import re

			




	
	
	
		
 
import itertools

			




	
	
	
		
 
import json

			




	
	
	
		
 
import gzip

			




	
	
	
		
 

			




	
	
	
		
 
TOP_NGRAM_COUNT = 3000

			




	
	
	
		
 
MODEL_PATH = os.path.join(os.path.dirname(__file__), "models.json.gz")

			




	
	
	
		
 

			




	
	
	
		
 

			




	
	
	
		
 
def preprocess(text):

			




	
	
	
		
 
	text = re.sub(r"[\W\d_]+", " ", " "+text+" ")

			




	
	
	
		
 
	return text.lower()

			




	
	
	
		
 

			




	
	
		
 
@@ -62,24 +66,28 @@ class Sample:

			




	
	
	
		
 
		return {

			




	
	
	
		
 
			"language": self.language,

			




	
	
	
		
 
			"ngrams": [key for (key, order) in sorted(self.ranked_ngrams.items(), key=lambda key_order: key_order[1])]

			




	
	
	
		
 
		}

			




	
	
	
		
 

			




	
	
	
		
 
	def compare(self, other):

			




	
	
	
		
 
		"""take k most common

			




	
	
	
		
 
		use frequencies x order

			




	
	
	
		
 
		use letter, digrams, trigrams

			




	
	
	
		
 
		use absolute x square"""

			




	
	
	
		
 
		m = len(other.ranked_ngrams)

			




	
	
	
		
 

			




	
	
	
		
 
		res = sum(

			




	
	
	
		
 
			(abs(v - other.ranked_ngrams[k]) if k in other.ranked_ngrams else m)

			




	
	
	
		
 
			for (k, v) in self.ranked_ngrams.items()

			




	
	
	
		
 
		)

			




	
	
	
		
 

			




	
	
	
		
 
		return res

			




	
	
	
		
 

			




	
	
	
		
 

			




	
	
	
		
 
def identify(text, models):

			




	
	
	
		
 
def load_models(model_path):

			




	
	
	
		
 
	with gzip.open(model_path, mode="rt", encoding="utf-8") as f:

			




	
	
	
		
 
		return [Sample.load(obj) for obj in json.load(f)]

			




	
	
	
		
 

			




	
	
	
		
 

			




	
	
	
		
 
def identify(text, models=[]):

			




	
	
	
		
 
	if not models:

			




	
	
	
		
 
		models = load_models(MODEL_PATH)

			




	
	
	
		
 

			




	
	
	
		
 
	sample = Sample.extract(text)

			




	
	
	
		
 

			




	
	
	
		
 
	return sorted(models, key=lambda m: sample.compare(m))[0].language

			




        

    



    


    



    



      

      





    
    0 comments (0 inline, 0 general)
    





    


  

  







  


    




    








    
        Server instance: Sirius-560
    
    
        This site is powered by
            Kallithea 0.7.0,
        which is
        © 2010–2023 by various authors & licensed under GPLv3.
            – Support