Changeset - fd790596c353
[Not reviewed]
default
0 2 0
Laman - 10 months ago 2024-06-30 16:15:19

implemented comparison of regexps with different alphabets
2 files changed with 65 insertions and 2 deletions:
0 comments (0 inline, 0 general)
regexp.py
Show inline comments
 
@@ -185,48 +185,56 @@ def parse(pattern, offset=0):
 
				token = res.pop()
 
			except IndexError as e:
 
				raise ParsingError(f'The asterisk operator is missing an argument. Pattern: "{pattern}", position {i}')
 
			res.append(Asterisk(token))
 
			i += 1
 
		elif c == ")":
 
			raise ParsingError(f'An opening parenthesis not found. Pattern: "{pattern}", position: {i}')
 
		elif c == "|" or c == "+":
 
			is_alternative = True
 
			res.append(AlternativeSeparator())
 
			i += 1
 
		elif c == "_":
 
			res.append(Lambda())
 
			i += 1
 
		else:
 
			res.append(Symbol(i+offset, c))
 
			i += 1
 

	
 
	if is_alternative:
 
		return Alternative(res)
 
	else:
 
		return Chain(res)
 

	
 

	
 
def print_dfa(dfa, label=""):
 
	n = len(dfa.alphabet_index)
 
	print(label)
 
	for i in range(0, len(dfa.rules), n):
 
		print(i//n, dfa.rules[i:i+n])
 
	print(dfa.end_states)
 

	
 

	
 
class Regexp:
 
	def __init__(self, pattern):
 
		r = parse(pattern)
 
		rules = dict()
 
		alphabet = set()
 

	
 
		for i in r.list_first():
 
			c = pattern[i]
 
			alphabet.add(c)
 
			key = (-1, c)
 
			if key not in rules:
 
				rules[key] = set()
 
			rules[key].add(i)
 

	
 
		for (i, j) in r.list_neighbours():
 
			c = pattern[j]
 
			alphabet.add(c)
 
			key = (i, c)
 
			if key not in rules:
 
				rules[key] = set()
 
			rules[key].add(j)
 

	
 
		end_states = set(r.list_last())
 
		if r.is_skippable:
 
@@ -319,49 +327,51 @@ class RegexpDFA:
 
	def normalize(self):
 
		n = len(self.alphabet_index)
 
		index = {0: 0}
 
		queue = deque([0])
 

	
 
		rules = []
 

	
 
		while queue:
 
			si = queue.popleft()
 
			row = self.rules[si*n:(si+1)*n]
 
			for sj in row:
 
				if sj not in index:
 
					index[sj] = len(index)
 
					queue.append(sj)
 
			rules.extend(index[sj] for sj in row)
 
		
 
		end_states = {index[si] for si in self.end_states if si in index}
 

	
 
		return RegexpDFA(rules, end_states, self.alphabet_index)
 

	
 
	def find_distinguishing_string(self, r):
 
		if self.rules == r.rules and self.end_states == r.end_states:
 
			return None
 

	
 
		product = self._build_product_automaton(r)
 
		r1 = self._expand_alphabet(r.alphabet_index)
 
		r2 = r._expand_alphabet(self.alphabet_index)
 
		product = r1._build_product_automaton(r2)
 

	
 
		n = len(product.alphabet_index)
 
		reverse_alphabet_index = {v: k for (k, v) in product.alphabet_index.items()}
 
		queue = deque([(0, "")])
 
		visited = {0}
 
		while queue:
 
			(state, acc) = queue.popleft()
 
			if state in product.end_states:
 
				return acc
 
			for (i, target) in enumerate(product.rules[state*n:(state+1)*n]):
 
				if target not in visited:
 
					queue.append((target, acc+reverse_alphabet_index[i]))
 
					visited.add(target)
 
		
 
		assert False
 

	
 
	def _find_equivalent_states(self):
 
		n = len(self.alphabet_index)
 
		state_list = list(range(len(self.rules) // n))
 
		equivalents = {(s1, s2) for (i, s1) in enumerate(state_list) for s2 in state_list[i+1:] if (s1 in self.end_states) == (s2 in self.end_states)}
 
		
 
		ctrl = True
 
		while ctrl:
 
			ctrl = False
 
@@ -380,48 +390,70 @@ class RegexpDFA:
 
	def _collapse_states(self, equivalents):
 
		n = len(self.alphabet_index)
 
		rules = []
 

	
 
		eq_mapping = dict()
 
		for (s1, s2) in equivalents:
 
			eq_mapping[s2] = min(s1, eq_mapping.get(s2, math.inf))
 

	
 
		discard_mapping = dict()
 
		discard_count = 0
 

	
 
		for i in range(0, len(self.rules), n):
 
			si = i//n
 
			if si in eq_mapping:
 
				discard_count += 1
 
				continue
 
			discard_mapping[si] = si - discard_count
 
			rules.extend(map(lambda st: eq_mapping.get(st, st), self.rules[i:i+n]))
 
		
 
		rules = [discard_mapping[st] for st in rules]
 
		end_states = {discard_mapping[eq_mapping.get(st, st)] for st in self.end_states}
 
		
 
		return (rules, end_states)
 

	
 
	def _expand_alphabet(self, alphabet_index):
 
		if alphabet_index == self.alphabet_index:
 
			return self
 

	
 
		n1 = len(self.alphabet_index)
 
		m = len(self.rules) // n1
 

	
 
		combined_alphabet = set(self.alphabet_index.keys()) | set(alphabet_index.keys())
 
		combined_index = {c: i for (i, c) in enumerate(sorted(combined_alphabet))}
 
		conversion_index = {v: combined_index[k] for (k, v) in self.alphabet_index.items()}
 
		n2 = len(combined_alphabet)
 

	
 
		rules = []
 
		for i in range(0, len(self.rules), n1):
 
			row = ([m]*n2)
 
			for (j, st) in enumerate(self.rules[i:i+n1]):
 
				row[conversion_index[j]] = st
 
			rules.extend(row)
 
		rules.extend([m]*n2)
 

	
 
		return RegexpDFA(rules, self.end_states, combined_index).reduce().normalize()
 

	
 
	def _build_product_automaton(self, r):
 
		n = len(self.alphabet_index)
 
		m = len(r.rules) // n
 
		k = len(self.rules) // n
 

	
 
		rules = []
 
		end_states = set()
 

	
 
		for s1 in range(k):
 
			row1 = self.rules[s1*n:(s1+1)*n]
 
			for s2 in range(m):
 
				row2 = r.rules[s2*n:(s2+1)*n]
 
				rules.extend([x*m + y for (x, y) in zip(row1, row2)])
 
				if (s1 in self.end_states) != (s2 in r.end_states):
 
					end_states.add(s1*m + s2)
 

	
 
		return RegexpDFA(rules, end_states, self.alphabet_index).reduce().normalize()
 

	
 

	
 
def test():
 
	tests = ["", "a", "ab", "aabb", "abab", "abcd", "abcbcdbcd"]
 
	for pattern in ["a(b|c)", "a*b*", "(ab)*", "a((bc)*d)*", "(a|b)*a(a|b)(a|b)(a|b)", "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz"]:
 
		print("#", pattern)
 
		try:
src/regexp.rs
Show inline comments
 
@@ -111,48 +111,49 @@ impl Regexp {
 
				let target_hash = encode_set(&target_set);
 
				let is_end = target_set.iter().any(|st| self.end_states.contains(st));
 
				if !index_new.contains_key(&target_hash) {
 
					let target_new = index_new.len();
 
					index_new.insert(target_hash.clone(), target_new);
 
					index_multi.insert(target_hash.clone(), target_set);
 
					compact_rules.extend(iter::repeat(FAIL).take(n));
 
					stack.push(target_hash.clone());
 
				}
 
				compact_rules[index_new[&state_hash]*n + alphabet_index[&c]] = index_new[&target_hash];
 
				if is_end {
 
					end_states.insert(index_new[&target_hash]);
 
				}
 
			}
 
		}
 

	
 
		let fail = index_new.len();
 
		compact_rules = compact_rules.into_iter().map(|st| if st != FAIL {st} else {fail}).collect();
 
		compact_rules.extend(iter::repeat(fail).take(n));
 

	
 
		return RegexpDFA{rules: compact_rules, end_states, alphabet_index};
 
	}
 
}
 

	
 
#[derive(Clone)]
 
pub struct RegexpDFA {
 
	rules: Vec<usize>,
 
	end_states: HashSet<usize>,
 
	alphabet_index: HashMap<char, usize>
 
}
 

	
 
impl RegexpDFA {
 
	pub fn eval(&self, s: String) -> bool {
 
		let n = self.alphabet_index.len();
 
		if n == 0 {
 
			return s.len() == 0;
 
		}
 
		let fail = self.rules.len() / n;
 
		let mut state = START_DFA;
 

	
 
		for c in s.chars() {
 
			if let Some(ci) = self.alphabet_index.get(&c) {
 
				state = self.rules[state*n + ci];
 
			} else {
 
				return false;
 
			}
 
			if state == fail {
 
				return false;
 
			}
 
@@ -185,49 +186,51 @@ impl RegexpDFA {
 

	
 
		while !queue.is_empty() {
 
			let si = queue.pop_front().unwrap();
 
			let row = &self.rules[si*n..(si+1)*n];
 
			for &sj in row {
 
				if sj != fail && index[sj] == fail {
 
					index[sj] = k;
 
					k += 1;
 
					queue.push_back(sj);
 
				}
 
			}
 
			rules.extend(row.iter().map(|&st| index[st]));
 
		}
 

	
 
		let end_states = self.end_states.iter().map(|st| index[*st]).collect();
 
		
 
		return RegexpDFA{rules, end_states, alphabet_index: self.alphabet_index.clone()};
 
	}
 

	
 
	pub fn find_distinguishing_string(&self, other: &RegexpDFA) -> Option<String> {
 
		if self.rules == other.rules && self.end_states == other.end_states {
 
			return None;
 
		}
 

	
 
		let product = self.build_product_automaton(other);
 
		let r1 = self.expand_alphabet(&other.alphabet_index);
 
		let r2 = other.expand_alphabet(&self.alphabet_index);
 
		let product = r1.build_product_automaton(&r2);
 
		let n = product.alphabet_index.len();
 
		let reverse_alphabet_index: HashMap<usize, char> = HashMap::from_iter(product.alphabet_index.iter().map(|(&k, &v)| (v, k)));
 

	
 
		let mut queue = VecDeque::from([(0, "".to_string())]);
 
		let mut visited = HashSet::new();
 
		while !queue.is_empty() {
 
			let (state, acc) = queue.pop_front().unwrap();
 
			if product.end_states.contains(&state) {
 
				return Some(acc);
 
			}
 
			for (i, target) in product.rules[state*n..(state+1)*n].iter().enumerate() {
 
				if !visited.contains(target) {
 
					queue.push_back((*target, acc.clone()+&String::from(reverse_alphabet_index[&i])));
 
					visited.insert(target);
 
				}
 
			}
 
		}
 

	
 
		panic!();
 
	}
 

	
 
	fn find_equivalent_states(&self) -> Vec<(usize, usize)> {
 
		let n = self.alphabet_index.len();
 
		let state_vec: Vec<usize> = (0..self.rules.len()/n).collect();
 
@@ -263,46 +266,74 @@ impl RegexpDFA {
 

	
 
		let mut eq_mapping: Vec<usize> = ((0..m)).collect();
 
		for (s1, s2) in equivalents.into_iter() {
 
			eq_mapping[s2] = eq_mapping[s2].min(s1);
 
		}
 

	
 
		let mut discard_mapping: Vec<usize> = ((0..m)).collect();
 
		let mut discard_count = 0;
 

	
 
		for si in 0..m {
 
			if eq_mapping[si] != si {
 
				discard_count += 1;
 
				continue;
 
			}
 
			discard_mapping[si] = si-discard_count;
 
			rules.extend(self.rules[si*n..(si+1)*n].iter().map(|&st| eq_mapping[st]));
 
		}
 

	
 
		rules = rules.into_iter().map(|st| discard_mapping[st]).collect();
 
		let end_states = self.end_states.iter().map(|st| discard_mapping[eq_mapping[*st]]).collect();
 

	
 
		return RegexpDFA{rules, end_states, alphabet_index: self.alphabet_index.clone()};
 
	}
 

	
 
	fn expand_alphabet(&self, alphabet_index: &HashMap<char, usize>) -> RegexpDFA {
 
		if *alphabet_index == self.alphabet_index {
 
			return self.clone();
 
		}
 

	
 
		let n1 = self.alphabet_index.len();
 
		let m = self.rules.len() / n1;
 

	
 
		let combined_alphabet: HashSet<char> = HashSet::from_iter(self.alphabet_index.keys().chain(alphabet_index.keys()).copied());
 
		let mut combined_vec = Vec::from_iter(combined_alphabet.into_iter());
 
		combined_vec.sort();
 
		let combined_index = HashMap::from_iter(combined_vec.iter().enumerate().map(|(i, c)| (*c, i)));
 
		let conversion_index: HashMap<usize, usize> = HashMap::from_iter(self.alphabet_index.iter().map(|(k, v)| (*v, combined_index[k])));
 
		let n2 = combined_vec.len();
 

	
 
		let mut rules = vec![];
 
		for i in 0..m {
 
			let mut row = vec![m;n2];
 
			for (j, st) in self.rules[i*n1..(i+1)*n1].iter().enumerate() {
 
				row[conversion_index[&j]] = *st;
 
			}
 
			rules.append(&mut row);
 
		}
 
		rules.append(&mut vec![m;n2]);
 

	
 
		return RegexpDFA{rules, end_states: self.end_states.clone(), alphabet_index: combined_index}.reduce().normalize();
 
	}
 

	
 
	fn build_product_automaton(&self, other: &RegexpDFA) -> RegexpDFA {
 
		let n = self.alphabet_index.len();
 
		let m = other.rules.len() / n;
 
		let k = self.rules.len() / n;
 

	
 
		let mut rules = vec![];
 
		let mut end_states = HashSet::new();
 

	
 
		for s1 in 0..k {
 
			let row1 = &self.rules[s1*n..(s1+1)*n];
 
			for s2 in 0..m {
 
				let row2 = &other.rules[s2*n..(s2+1)*n];
 
				rules.extend(row1.iter().zip(row2.iter()).map(|(x, y)| x*m + y));
 
				if (self.end_states.contains(&s1)) ^ (other.end_states.contains(&s2)) {
 
					end_states.insert(s1*m + s2);
 
				}
 
			}
 
		}
 

	
 
		return RegexpDFA{rules, end_states, alphabet_index: self.alphabet_index.clone()}.reduce().normalize();
 
	}
 
}
0 comments (0 inline, 0 general)