Changeset - 1790bcb433e3
[Not reviewed]
default
0 2 0
Laman - 10 months ago 2024-06-30 17:02:12

fixes: never create an empty automaton, removed the erroneous early fail
2 files changed with 18 insertions and 20 deletions:
0 comments (0 inline, 0 general)
regexp.py
Show inline comments
 
@@ -249,114 +249,113 @@ class Regexp:
 

	
 
		for c in s:
 
			new_state = set()
 
			for st in current:
 
				key = (st, c)
 
				if key in self.rules:
 
					new_state.update(self.rules[key])
 
			current = new_state
 

	
 
		return any(st in self.end_states for st in current)
 

	
 
	def determinize(self):
 
		alphabet_index = {c: i for (i, c) in enumerate(sorted(self.alphabet))}
 
		n = len(alphabet_index)
 
		compact_rules = [-1] * n
 
		end_states = {0} if -1 in self.end_states else set()
 

	
 
		index = {(-1,): 0}
 
		stack = [(-1,)]
 
		while stack:
 
			multistate = stack.pop()
 
			new_rules = dict()
 
			
 
			for ((st, c), target) in filter(lambda item: item[0][0] in multistate, self.rules.items()):
 
				if c not in new_rules:
 
					new_rules[c] = set()
 
				new_rules[c].update(target)
 
			
 
			for (c, target_set) in new_rules.items():
 
				target_tup = tuple(sorted(target_set))
 
				if target_tup not in index:
 
					new_target = len(index)
 
					index[target_tup] = new_target
 
					compact_rules.extend([-1] * n)
 
					stack.append(target_tup)
 
				compact_rules[index[multistate]*n + alphabet_index[c]] = index[target_tup]
 
				if any(st in self.end_states for st in target_set):
 
					end_states.add(index[target_tup])
 

	
 
		fail = len(index)
 
		compact_rules = [(st if st >= 0 else fail) for st in compact_rules]
 
		compact_rules.extend([fail] * n)
 
		
 
		return (compact_rules, end_states, alphabet_index)
 

	
 

	
 
class RegexpDFA:
 
	def __init__(self, rules, end_states, alphabet_index):
 
		self.rules = rules
 
		self.rules = rules or [1, 1]
 
		self.end_states = end_states
 
		self.alphabet_index = alphabet_index
 
		self.alphabet_index = alphabet_index or {"": 0}
 

	
 
	@classmethod
 
	def create(cls, pattern):
 
		r = Regexp(pattern)
 
		(rules, end_states, alphabet_index) = r.determinize()
 

	
 
		return cls(rules, end_states, alphabet_index)
 

	
 
	def match(self, s):
 
		st = 0
 
		n = len(self.alphabet_index)
 
		fail = len(self.rules) // n
 

	
 
		for c in s:
 
			if c not in self.alphabet_index or st == fail:
 
			if c not in self.alphabet_index:
 
				return False
 
			key = (st*n + self.alphabet_index[c])
 
			st = self.rules[key]
 

	
 
		return st in self.end_states
 

	
 
	def reduce(self):
 
		equivalents = self._find_equivalent_states()
 
		(rules, end_states) = self._collapse_states(equivalents)
 

	
 
		return RegexpDFA(rules, end_states, self.alphabet_index)
 

	
 
	def normalize(self):
 
		n = len(self.alphabet_index)
 
		index = {0: 0}
 
		queue = deque([0])
 

	
 
		rules = []
 

	
 
		while queue:
 
			si = queue.popleft()
 
			row = self.rules[si*n:(si+1)*n]
 
			for sj in row:
 
				if sj not in index:
 
					index[sj] = len(index)
 
					queue.append(sj)
 
			rules.extend(index[sj] for sj in row)
 
		
 
		end_states = {index[si] for si in self.end_states if si in index}
 

	
 
		return RegexpDFA(rules, end_states, self.alphabet_index)
 

	
 
	def find_distinguishing_string(self, r):
 
		if self.rules == r.rules and self.end_states == r.end_states:
 
			return None
 

	
 
		r1 = self._expand_alphabet(r.alphabet_index)
 
		r2 = r._expand_alphabet(self.alphabet_index)
 
		product = r1._build_product_automaton(r2)
 

	
 
		n = len(product.alphabet_index)
 
		reverse_alphabet_index = {v: k for (k, v) in product.alphabet_index.items()}
 
		queue = deque([(0, "")])
 
		visited = {0}
 
		while queue:
 
			(state, acc) = queue.popleft()
 
			if state in product.end_states:
 
				return acc
src/regexp.rs
Show inline comments
 
@@ -52,174 +52,173 @@ impl Regexp {
 
		if r.is_skippable() {
 
			end_states.insert(START_NFA);
 
		}
 

	
 
		let mut alphabet_vec = Vec::from_iter(alphabet.into_iter());
 
		alphabet_vec.sort();
 

	
 
		return Ok(Regexp{rules, end_states, alphabet: alphabet_vec});
 
	}
 

	
 
	pub fn eval(&self, s: String) -> bool {
 
		let mut multistate = HashSet::from([START_NFA]);
 

	
 
		for c in s.chars() {
 
			let mut new_multistate = HashSet::new();
 

	
 
			for state in multistate {
 
				if let Some(x) = self.rules.get(&(state, c)) {
 
					new_multistate = new_multistate.union(&x).map(|&y| y).collect();
 
				} else if let Some(x) = self.rules.get(&(state, '.')) {
 
					new_multistate = new_multistate.union(&x).map(|&y| y).collect();
 
				}
 
			}
 
			multistate = new_multistate;
 
		}
 

	
 
		return multistate.iter().any(|x| self.end_states.contains(x));
 
	}
 

	
 
	pub fn determinize(&self) -> RegexpDFA {
 
		const FAIL: usize = usize::MAX;
 
		let alphabet_index: HashMap<char, usize> = self.alphabet.iter().enumerate().map(|(i, c)| (*c, i)).collect();
 
		let n = alphabet_index.len();
 
		let mut compact_rules = vec![FAIL; n];
 
		let mut end_states: HashSet<usize> = HashSet::new();
 
		if self.end_states.contains(&START_NFA) {end_states.insert(START_DFA);}
 

	
 
		// string hash -> single int DFA state
 
		let mut index_new = HashMap::from([(START_NFA.to_string(), START_DFA)]);
 
		// string hash -> HashSet NFA multistate
 
		let mut index_multi = HashMap::from([(START_NFA.to_string(), HashSet::from([START_NFA]))]);
 
		let mut stack = Vec::from([START_NFA.to_string()]);
 

	
 
		while !stack.is_empty() {
 
			let state_hash = stack.pop().unwrap();
 
			let multistate = &index_multi[&state_hash];
 
			let mut new_rules: HashMap<char, HashSet<usize>> = HashMap::new();
 

	
 
			for key in self.rules.keys().filter(|key| multistate.contains(&key.0)) {
 
			for key in self.rules.keys().filter(|(st, _c)| multistate.contains(st)) {
 
				let (_st, c) = key;
 
				if !new_rules.contains_key(c) {
 
					new_rules.insert(*c, HashSet::new());
 
				}
 
				for target in &self.rules[key] {
 
					new_rules.get_mut(c).unwrap().insert(*target);
 
				}
 
			}
 

	
 
			for (c, target_set) in new_rules.into_iter() {
 
				let target_hash = encode_set(&target_set);
 
				let is_end = target_set.iter().any(|st| self.end_states.contains(st));
 
				if !index_new.contains_key(&target_hash) {
 
					let target_new = index_new.len();
 
					index_new.insert(target_hash.clone(), target_new);
 
					index_multi.insert(target_hash.clone(), target_set);
 
					compact_rules.extend(iter::repeat(FAIL).take(n));
 
					stack.push(target_hash.clone());
 
				}
 
				compact_rules[index_new[&state_hash]*n + alphabet_index[&c]] = index_new[&target_hash];
 
				if is_end {
 
					end_states.insert(index_new[&target_hash]);
 
				}
 
			}
 
		}
 

	
 
		let fail = index_new.len();
 
		compact_rules = compact_rules.into_iter().map(|st| if st != FAIL {st} else {fail}).collect();
 
		compact_rules.extend(iter::repeat(fail).take(n));
 

	
 
		return RegexpDFA{rules: compact_rules, end_states, alphabet_index};
 
		
 
		return RegexpDFA::new(compact_rules, end_states, alphabet_index);
 
	}
 
}
 

	
 
#[derive(Clone)]
 
pub struct RegexpDFA {
 
	rules: Vec<usize>,
 
	end_states: HashSet<usize>,
 
	alphabet_index: HashMap<char, usize>
 
}
 

	
 
impl RegexpDFA {
 
	pub fn new(rules: Vec<usize>, end_states: HashSet<usize>, alphabet_index: HashMap<char, usize>) -> RegexpDFA {
 
		if rules.len() > 0 {
 
			return RegexpDFA{rules, end_states, alphabet_index};
 
		} else {
 
			return RegexpDFA{
 
				rules: vec![1, 1],
 
				end_states,
 
				alphabet_index: HashMap::from([('\0', 0)])
 
			};
 
		}
 
	}
 

	
 
	pub fn eval(&self, s: String) -> bool {
 
		let n = self.alphabet_index.len();
 
		if n == 0 {
 
			return s.len() == 0;
 
		}
 
		let fail = self.rules.len() / n;
 
		let mut state = START_DFA;
 

	
 
		for c in s.chars() {
 
			if let Some(ci) = self.alphabet_index.get(&c) {
 
				state = self.rules[state*n + ci];
 
			} else {
 
				return false;
 
			}
 
			if state == fail {
 
				return false;
 
			}
 
		}
 

	
 
		return self.end_states.contains(&state);
 
	}
 

	
 
	pub fn reduce(&self) -> RegexpDFA {
 
		if self.alphabet_index.len() == 0 {
 
			return RegexpDFA{rules: self.rules.clone(), end_states: self.end_states.clone(), alphabet_index: self.alphabet_index.clone()};
 
		}
 
		let equivalents = self.find_equivalent_states();
 
		return self.collapse_states(equivalents);
 
	}
 

	
 
	pub fn normalize(&self) -> RegexpDFA {
 
		let n = self.alphabet_index.len();
 
		if n == 0 {
 
			return RegexpDFA{rules: self.rules.clone(), end_states: self.end_states.clone(), alphabet_index: self.alphabet_index.clone()}; 
 
		}
 
		let m = self.rules.len()/n;
 
		let fail = m;
 
		let mut index: Vec<usize> = vec![fail;m];
 
		index[0] = 0;
 
		let mut queue = VecDeque::from([START_DFA]);
 

	
 
		let mut rules = vec![];
 
		let mut k = 1;
 

	
 
		while !queue.is_empty() {
 
			let si = queue.pop_front().unwrap();
 
			let row = &self.rules[si*n..(si+1)*n];
 
			for &sj in row {
 
				if sj != fail && index[sj] == fail {
 
					index[sj] = k;
 
					k += 1;
 
					queue.push_back(sj);
 
				}
 
			}
 
			rules.extend(row.iter().map(|&st| index[st]));
 
		}
 

	
 
		let end_states = self.end_states.iter().map(|st| index[*st]).collect();
 
		
 
		return RegexpDFA{rules, end_states, alphabet_index: self.alphabet_index.clone()};
 
	}
 

	
 
	pub fn find_distinguishing_string(&self, other: &RegexpDFA) -> Option<String> {
 
		if self.rules == other.rules && self.end_states == other.end_states {
 
			return None;
 
		}
 

	
 
		let r1 = self.expand_alphabet(&other.alphabet_index);
 
		let r2 = other.expand_alphabet(&self.alphabet_index);
 
		let product = r1.build_product_automaton(&r2);
 
		let n = product.alphabet_index.len();
 
		let reverse_alphabet_index: HashMap<usize, char> = HashMap::from_iter(product.alphabet_index.iter().map(|(&k, &v)| (v, k)));
 

	
 
		let mut queue = VecDeque::from([(0, "".to_string())]);
 
		let mut visited = HashSet::new();
 
		while !queue.is_empty() {
 
			let (state, acc) = queue.pop_front().unwrap();
 
			if product.end_states.contains(&state) {
 
				return Some(acc);
 
			}
 
			for (i, target) in product.rules[state*n..(state+1)*n].iter().enumerate() {
 
				if !visited.contains(target) {
 
					queue.push_back((*target, acc.clone()+&String::from(reverse_alphabet_index[&i])));
0 comments (0 inline, 0 general)