build(rust): vendor googletest crate
This commit was merged in pull request #76.
This commit is contained in:
1 parent
a6fe84190e
commit
6887b104d6
576 files changed
+284290
-2
No files matched your search
@@ -0,0 +1,162 @@
|
||||
use std::error::Error;
|
||||
|
||||
use regex_automata::{
|
||||
dfa::{dense, Automaton, OverlappingState},
|
||||
nfa::thompson,
|
||||
Anchored, HalfMatch, Input, MatchError,
|
||||
};
|
||||
|
||||
// Tests that quit bytes in the forward direction work correctly.
|
||||
#[test]
|
||||
fn quit_fwd() -> Result<(), Box<dyn Error>> {
|
||||
let dfa = dense::Builder::new()
|
||||
.configure(dense::Config::new().quit(b'x', true))
|
||||
.build("[[:word:]]+$")?;
|
||||
|
||||
assert_eq!(
|
||||
Err(MatchError::quit(b'x', 3)),
|
||||
dfa.try_search_fwd(&Input::new(b"abcxyz"))
|
||||
);
|
||||
assert_eq!(
|
||||
dfa.try_search_overlapping_fwd(
|
||||
&Input::new(b"abcxyz"),
|
||||
&mut OverlappingState::start()
|
||||
),
|
||||
Err(MatchError::quit(b'x', 3)),
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// Tests that quit bytes in the reverse direction work correctly.
|
||||
#[test]
|
||||
fn quit_rev() -> Result<(), Box<dyn Error>> {
|
||||
let dfa = dense::Builder::new()
|
||||
.configure(dense::Config::new().quit(b'x', true))
|
||||
.thompson(thompson::Config::new().reverse(true))
|
||||
.build("^[[:word:]]+")?;
|
||||
|
||||
assert_eq!(
|
||||
Err(MatchError::quit(b'x', 3)),
|
||||
dfa.try_search_rev(&Input::new(b"abcxyz"))
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// Tests that if we heuristically enable Unicode word boundaries but then
|
||||
// instruct that a non-ASCII byte should NOT be a quit byte, then the builder
|
||||
// will panic.
|
||||
#[test]
|
||||
#[should_panic]
|
||||
fn quit_panics() {
|
||||
dense::Config::new().unicode_word_boundary(true).quit(b'\xFF', false);
|
||||
}
|
||||
|
||||
// This tests an intesting case where even if the Unicode word boundary option
|
||||
// is disabled, setting all non-ASCII bytes to be quit bytes will cause Unicode
|
||||
// word boundaries to be enabled.
|
||||
#[test]
|
||||
fn unicode_word_implicitly_works() -> Result<(), Box<dyn Error>> {
|
||||
let mut config = dense::Config::new();
|
||||
for b in 0x80..=0xFF {
|
||||
config = config.quit(b, true);
|
||||
}
|
||||
let dfa = dense::Builder::new().configure(config).build(r"\b")?;
|
||||
let expected = HalfMatch::must(0, 1);
|
||||
assert_eq!(Ok(Some(expected)), dfa.try_search_fwd(&Input::new(b" a")));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// A variant of [`Automaton::is_special_state`]'s doctest, but with universal
|
||||
// start states.
|
||||
//
|
||||
// See: https://github.com/rust-lang/regex/pull/1195
|
||||
#[test]
|
||||
fn universal_start_search() -> Result<(), Box<dyn Error>> {
|
||||
fn find<A: Automaton>(
|
||||
dfa: &A,
|
||||
haystack: &[u8],
|
||||
) -> Result<Option<HalfMatch>, MatchError> {
|
||||
let mut state = dfa
|
||||
.universal_start_state(Anchored::No)
|
||||
.expect("regex should not require lookbehind");
|
||||
let mut last_match = None;
|
||||
// Walk all the bytes in the haystack. We can quit early if we see
|
||||
// a dead or a quit state. The former means the automaton will
|
||||
// never transition to any other state. The latter means that the
|
||||
// automaton entered a condition in which its search failed.
|
||||
for (i, &b) in haystack.iter().enumerate() {
|
||||
state = dfa.next_state(state, b);
|
||||
if dfa.is_special_state(state) {
|
||||
if dfa.is_match_state(state) {
|
||||
last_match =
|
||||
Some(HalfMatch::new(dfa.match_pattern(state, 0), i));
|
||||
} else if dfa.is_dead_state(state) {
|
||||
return Ok(last_match);
|
||||
} else if dfa.is_quit_state(state) {
|
||||
// It is possible to enter into a quit state after
|
||||
// observing a match has occurred. In that case, we
|
||||
// should return the match instead of an error.
|
||||
if last_match.is_some() {
|
||||
return Ok(last_match);
|
||||
}
|
||||
return Err(MatchError::quit(b, i));
|
||||
}
|
||||
// Implementors may also want to check for start or accel
|
||||
// states and handle them differently for performance
|
||||
// reasons. But it is not necessary for correctness.
|
||||
}
|
||||
}
|
||||
// Matches are always delayed by 1 byte, so we must explicitly walk
|
||||
// the special "EOI" transition at the end of the search.
|
||||
state = dfa.next_eoi_state(state);
|
||||
if dfa.is_match_state(state) {
|
||||
last_match = Some(HalfMatch::new(
|
||||
dfa.match_pattern(state, 0),
|
||||
haystack.len(),
|
||||
));
|
||||
}
|
||||
Ok(last_match)
|
||||
}
|
||||
|
||||
fn check_impl(
|
||||
dfa: impl Automaton,
|
||||
haystack: &str,
|
||||
pat: usize,
|
||||
offset: usize,
|
||||
) -> Result<(), Box<dyn Error>> {
|
||||
let haystack = haystack.as_bytes();
|
||||
let mat = find(&dfa, haystack)?.unwrap();
|
||||
assert_eq!(mat.pattern().as_usize(), pat);
|
||||
assert_eq!(mat.offset(), offset);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn check(
|
||||
dfa: &dense::DFA<Vec<u32>>,
|
||||
haystack: &str,
|
||||
pat: usize,
|
||||
offset: usize,
|
||||
) -> Result<(), Box<dyn Error>> {
|
||||
check_impl(dfa, haystack, pat, offset)?;
|
||||
check_impl(dfa.to_sparse()?, haystack, pat, offset)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
let dfa = dense::DFA::new(r"[a-z]+")?;
|
||||
let haystack = "123 foobar 4567";
|
||||
check(&dfa, haystack, 0, 10)?;
|
||||
|
||||
let dfa = dense::DFA::new(r"[0-9]{4}")?;
|
||||
let haystack = "123 foobar 4567";
|
||||
check(&dfa, haystack, 0, 15)?;
|
||||
|
||||
let dfa = dense::DFA::new_many(&[r"[a-z]+", r"[0-9]+"])?;
|
||||
let haystack = "123 foobar 4567";
|
||||
check(&dfa, haystack, 1, 3)?;
|
||||
check(&dfa, &haystack[3..], 0, 7)?;
|
||||
check(&dfa, &haystack[10..], 1, 5)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
#[cfg(all(feature = "dfa-build", feature = "dfa-search"))]
|
||||
mod api;
|
||||
#[cfg(feature = "dfa-onepass")]
|
||||
mod onepass;
|
||||
#[cfg(all(feature = "dfa-build", feature = "dfa-search"))]
|
||||
mod regression;
|
||||
#[cfg(all(not(miri), feature = "dfa-build", feature = "dfa-search"))]
|
||||
mod suite;
|
||||
@@ -0,0 +1,3 @@
|
||||
mod regression;
|
||||
#[cfg(not(miri))]
|
||||
mod suite;
|
||||
@@ -0,0 +1,61 @@
|
||||
// Regression test for zero-repetition capture groups,
|
||||
// which caused a panic when the Vec passed into search_slots
|
||||
// contained space for the capture group which would never
|
||||
// have any results.
|
||||
//
|
||||
// See: https://github.com/rust-lang/regex/issues/1327
|
||||
#[test]
|
||||
fn zero_repetition_capture_group() {
|
||||
use regex_automata::{
|
||||
dfa::onepass::DFA, util::primitives::NonMaxUsize, Anchored, Input,
|
||||
};
|
||||
|
||||
let expr = DFA::new(r"(abc)(ABC){0}").unwrap();
|
||||
let s = "abcABC";
|
||||
let input = Input::new(s).span(0..s.len()).anchored(Anchored::Yes);
|
||||
|
||||
// Test with 4 slots, so the whole match plus the first capture group.
|
||||
let mut cache = expr.create_cache();
|
||||
let mut slots: Vec<Option<NonMaxUsize>> = vec![None; 4];
|
||||
let pid = expr.try_search_slots(&mut cache, &input, &mut slots).unwrap();
|
||||
assert_eq!(pid, Some(regex_automata::PatternID::must(0)));
|
||||
assert_eq!(slots[0], Some(NonMaxUsize::new(0).unwrap()));
|
||||
assert_eq!(slots[1], Some(NonMaxUsize::new(3).unwrap()));
|
||||
assert_eq!(slots[2], Some(NonMaxUsize::new(0).unwrap()));
|
||||
assert_eq!(slots[3], Some(NonMaxUsize::new(3).unwrap()));
|
||||
|
||||
// Test with larger slot array, which would fit the
|
||||
// zero-repetition capture group.
|
||||
slots.resize(6, None);
|
||||
let pid = expr.try_search_slots(&mut cache, &input, &mut slots).unwrap();
|
||||
assert_eq!(pid, Some(regex_automata::PatternID::must(0)));
|
||||
// First capture group should match
|
||||
assert_eq!(slots[2], Some(NonMaxUsize::new(0).unwrap()));
|
||||
assert_eq!(slots[3], Some(NonMaxUsize::new(3).unwrap()));
|
||||
// Second capture group with {0} should be None.
|
||||
assert_eq!(slots[4], None);
|
||||
assert_eq!(slots[5], None);
|
||||
}
|
||||
|
||||
// Another regression test for the same case as
|
||||
// `zero_repetition_capture_group`, but uses a simpler pattern. That
|
||||
// is, a zero-repetition capture group is a red herring. The actual bug
|
||||
// is simpler: it happens whenever too many slots are provided by the
|
||||
// caller.
|
||||
#[test]
|
||||
fn too_many_slots_normal_pattern() {
|
||||
use regex_automata::{
|
||||
dfa::onepass::DFA, util::primitives::NonMaxUsize, Anchored, Input,
|
||||
};
|
||||
|
||||
let expr = DFA::new(r"abc").unwrap();
|
||||
let s = "abc";
|
||||
let input = Input::new(s).span(0..s.len()).anchored(Anchored::Yes);
|
||||
|
||||
let mut cache = expr.create_cache();
|
||||
let mut slots: Vec<Option<NonMaxUsize>> = vec![None; 4];
|
||||
let pid = expr.try_search_slots(&mut cache, &input, &mut slots).unwrap();
|
||||
assert_eq!(pid, Some(regex_automata::PatternID::must(0)));
|
||||
assert_eq!(slots[0], Some(NonMaxUsize::new(0).unwrap()));
|
||||
assert_eq!(slots[1], Some(NonMaxUsize::new(3).unwrap()));
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
use {
|
||||
anyhow::Result,
|
||||
regex_automata::{
|
||||
dfa::onepass::{self, DFA},
|
||||
nfa::thompson,
|
||||
util::{iter, syntax},
|
||||
},
|
||||
regex_test::{
|
||||
CompiledRegex, Match, RegexTest, SearchKind, Span, TestResult,
|
||||
TestRunner,
|
||||
},
|
||||
};
|
||||
|
||||
use crate::{create_input, suite, testify_captures, untestify_kind};
|
||||
|
||||
const EXPANSIONS: &[&str] = &["is_match", "find", "captures"];
|
||||
|
||||
/// Tests the default configuration of the hybrid NFA/DFA.
|
||||
#[test]
|
||||
fn default() -> Result<()> {
|
||||
let builder = DFA::builder();
|
||||
TestRunner::new()?
|
||||
.expand(EXPANSIONS, |t| t.compiles())
|
||||
.test_iter(suite()?.iter(), compiler(builder))
|
||||
.assert();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Tests the hybrid NFA/DFA when 'starts_for_each_pattern' is enabled for all
|
||||
/// tests.
|
||||
#[test]
|
||||
fn starts_for_each_pattern() -> Result<()> {
|
||||
let mut builder = DFA::builder();
|
||||
builder.configure(DFA::config().starts_for_each_pattern(true));
|
||||
TestRunner::new()?
|
||||
.expand(EXPANSIONS, |t| t.compiles())
|
||||
.test_iter(suite()?.iter(), compiler(builder))
|
||||
.assert();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Tests the hybrid NFA/DFA when byte classes are disabled.
|
||||
///
|
||||
/// N.B. Disabling byte classes doesn't avoid any indirection at search time.
|
||||
/// All it does is cause every byte value to be its own distinct equivalence
|
||||
/// class.
|
||||
#[test]
|
||||
fn no_byte_classes() -> Result<()> {
|
||||
let mut builder = DFA::builder();
|
||||
builder.configure(DFA::config().byte_classes(false));
|
||||
TestRunner::new()?
|
||||
.expand(EXPANSIONS, |t| t.compiles())
|
||||
.test_iter(suite()?.iter(), compiler(builder))
|
||||
.assert();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn compiler(
|
||||
mut builder: onepass::Builder,
|
||||
) -> impl FnMut(&RegexTest, &[String]) -> Result<CompiledRegex> {
|
||||
move |test, regexes| {
|
||||
// Check if our regex contains things that aren't supported by DFAs.
|
||||
// That is, Unicode word boundaries when searching non-ASCII text.
|
||||
if !configure_onepass_builder(test, &mut builder) {
|
||||
return Ok(CompiledRegex::skip());
|
||||
}
|
||||
let re = match builder.build_many(®exes) {
|
||||
Ok(re) => re,
|
||||
Err(err) => {
|
||||
let msg = err.to_string();
|
||||
// This is pretty gross, but when a regex fails to compile as
|
||||
// a one-pass regex, then we want to be OK with that and just
|
||||
// skip the test. But we have to be careful to only skip it
|
||||
// when the expected result is that the regex compiles. If
|
||||
// the test is specifically checking that the regex does not
|
||||
// compile, then we should bubble up that error and allow the
|
||||
// test to pass.
|
||||
//
|
||||
// Since our error types are all generally opaque, we just
|
||||
// look for an error string. Not great, but not the end of the
|
||||
// world.
|
||||
if test.compiles() && msg.contains("not one-pass") {
|
||||
return Ok(CompiledRegex::skip());
|
||||
}
|
||||
return Err(err.into());
|
||||
}
|
||||
};
|
||||
let mut cache = re.create_cache();
|
||||
Ok(CompiledRegex::compiled(move |test| -> TestResult {
|
||||
run_test(&re, &mut cache, test)
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
fn run_test(
|
||||
re: &DFA,
|
||||
cache: &mut onepass::Cache,
|
||||
test: &RegexTest,
|
||||
) -> TestResult {
|
||||
let input = create_input(test);
|
||||
match test.additional_name() {
|
||||
"is_match" => {
|
||||
TestResult::matched(re.is_match(cache, input.earliest(true)))
|
||||
}
|
||||
"find" => match test.search_kind() {
|
||||
SearchKind::Earliest | SearchKind::Leftmost => {
|
||||
let input =
|
||||
input.earliest(test.search_kind() == SearchKind::Earliest);
|
||||
let mut caps = re.create_captures();
|
||||
let it = iter::Searcher::new(input)
|
||||
.into_matches_iter(|input| {
|
||||
re.try_search(cache, input, &mut caps)?;
|
||||
Ok(caps.get_match())
|
||||
})
|
||||
.infallible()
|
||||
.take(test.match_limit().unwrap_or(std::usize::MAX))
|
||||
.map(|m| Match {
|
||||
id: m.pattern().as_usize(),
|
||||
span: Span { start: m.start(), end: m.end() },
|
||||
});
|
||||
TestResult::matches(it)
|
||||
}
|
||||
SearchKind::Overlapping => {
|
||||
// The one-pass DFA does not support any kind of overlapping
|
||||
// search. This is not just a matter of not having the API.
|
||||
// It's fundamentally incompatible with the one-pass concept.
|
||||
// If overlapping matches were possible, then the one-pass DFA
|
||||
// would fail to build.
|
||||
TestResult::skip()
|
||||
}
|
||||
},
|
||||
"captures" => match test.search_kind() {
|
||||
SearchKind::Earliest | SearchKind::Leftmost => {
|
||||
let input =
|
||||
input.earliest(test.search_kind() == SearchKind::Earliest);
|
||||
let it = iter::Searcher::new(input)
|
||||
.into_captures_iter(re.create_captures(), |input, caps| {
|
||||
re.try_search(cache, input, caps)
|
||||
})
|
||||
.infallible()
|
||||
.take(test.match_limit().unwrap_or(std::usize::MAX))
|
||||
.map(|caps| testify_captures(&caps));
|
||||
TestResult::captures(it)
|
||||
}
|
||||
SearchKind::Overlapping => {
|
||||
// The one-pass DFA does not support any kind of overlapping
|
||||
// search. This is not just a matter of not having the API.
|
||||
// It's fundamentally incompatible with the one-pass concept.
|
||||
// If overlapping matches were possible, then the one-pass DFA
|
||||
// would fail to build.
|
||||
TestResult::skip()
|
||||
}
|
||||
},
|
||||
name => TestResult::fail(&format!("unrecognized test name: {name}")),
|
||||
}
|
||||
}
|
||||
|
||||
/// Configures the given regex builder with all relevant settings on the given
|
||||
/// regex test.
|
||||
///
|
||||
/// If the regex test has a setting that is unsupported, then this returns
|
||||
/// false (implying the test should be skipped).
|
||||
fn configure_onepass_builder(
|
||||
test: &RegexTest,
|
||||
builder: &mut onepass::Builder,
|
||||
) -> bool {
|
||||
if !test.anchored() {
|
||||
return false;
|
||||
}
|
||||
let match_kind = match untestify_kind(test.match_kind()) {
|
||||
None => return false,
|
||||
Some(k) => k,
|
||||
};
|
||||
|
||||
let config = DFA::config().match_kind(match_kind);
|
||||
builder
|
||||
.configure(config)
|
||||
.syntax(config_syntax(test))
|
||||
.thompson(config_thompson(test));
|
||||
true
|
||||
}
|
||||
|
||||
/// Configuration of a Thompson NFA compiler from a regex test.
|
||||
fn config_thompson(test: &RegexTest) -> thompson::Config {
|
||||
let mut lookm = regex_automata::util::look::LookMatcher::new();
|
||||
lookm.set_line_terminator(test.line_terminator());
|
||||
thompson::Config::new().utf8(test.utf8()).look_matcher(lookm)
|
||||
}
|
||||
|
||||
/// Configuration of the regex parser from a regex test.
|
||||
fn config_syntax(test: &RegexTest) -> syntax::Config {
|
||||
syntax::Config::new()
|
||||
.case_insensitive(test.case_insensitive())
|
||||
.unicode(test.unicode())
|
||||
.utf8(test.utf8())
|
||||
.line_terminator(test.line_terminator())
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
// A regression test for checking that minimization correctly translates
|
||||
// whether a state is a match state or not. Previously, it was possible for
|
||||
// minimization to mark a non-matching state as matching.
|
||||
#[test]
|
||||
#[cfg(not(miri))]
|
||||
fn minimize_sets_correct_match_states() {
|
||||
use regex_automata::{
|
||||
dfa::{dense::DFA, Automaton, StartKind},
|
||||
Anchored, Input,
|
||||
};
|
||||
|
||||
let pattern =
|
||||
// This is a subset of the grapheme matching regex. I couldn't seem
|
||||
// to get a repro any smaller than this unfortunately.
|
||||
r"(?x)
|
||||
(?:
|
||||
\p{gcb=Prepend}*
|
||||
(?:
|
||||
(?:
|
||||
(?:
|
||||
\p{gcb=L}*
|
||||
(?:\p{gcb=V}+|\p{gcb=LV}\p{gcb=V}*|\p{gcb=LVT})
|
||||
\p{gcb=T}*
|
||||
)
|
||||
|
|
||||
\p{gcb=L}+
|
||||
|
|
||||
\p{gcb=T}+
|
||||
)
|
||||
|
|
||||
\p{Extended_Pictographic}
|
||||
(?:\p{gcb=Extend}*\p{gcb=ZWJ}\p{Extended_Pictographic})*
|
||||
|
|
||||
[^\p{gcb=Control}\p{gcb=CR}\p{gcb=LF}]
|
||||
)
|
||||
[\p{gcb=Extend}\p{gcb=ZWJ}\p{gcb=SpacingMark}]*
|
||||
)
|
||||
";
|
||||
|
||||
let dfa = DFA::builder()
|
||||
.configure(
|
||||
DFA::config().start_kind(StartKind::Anchored).minimize(true),
|
||||
)
|
||||
.build(pattern)
|
||||
.unwrap();
|
||||
let input = Input::new(b"\xE2").anchored(Anchored::Yes);
|
||||
assert_eq!(Ok(None), dfa.try_search_fwd(&input));
|
||||
}
|
||||
|
||||
// A prefilter must not skip to a new candidate after the DFA has already
|
||||
// recorded a match. The DFA is still resolving the priority of that match,
|
||||
// and skipping can cause a later match to overwrite it.
|
||||
#[test]
|
||||
#[cfg(feature = "perf-literal-multisubstring")]
|
||||
fn prefilter_does_not_skip_pending_match() {
|
||||
use regex_automata::{
|
||||
dfa::{dense::DFA, Automaton},
|
||||
util::prefilter::Prefilter,
|
||||
HalfMatch, Input, MatchKind,
|
||||
};
|
||||
|
||||
let pre = Prefilter::new(MatchKind::LeftmostFirst, &["ab", "c"])
|
||||
.expect("prefilter");
|
||||
let dfa = DFA::builder()
|
||||
.configure(
|
||||
DFA::config().prefilter(Some(pre)).starts_for_each_pattern(true),
|
||||
)
|
||||
.build(r"(?:ab|c)*c")
|
||||
.unwrap();
|
||||
let expected = Ok(Some(HalfMatch::must(0, 1)));
|
||||
let input = Input::new("cabac");
|
||||
|
||||
assert_eq!(expected, dfa.try_search_fwd(&input));
|
||||
assert_eq!(expected, dfa.to_sparse().unwrap().try_search_fwd(&input));
|
||||
}
|
||||
@@ -0,0 +1,443 @@
|
||||
use {
|
||||
anyhow::Result,
|
||||
regex_automata::{
|
||||
dfa::{
|
||||
self, dense, regex::Regex, sparse, Automaton, OverlappingState,
|
||||
StartKind,
|
||||
},
|
||||
nfa::thompson,
|
||||
util::{prefilter::Prefilter, syntax},
|
||||
Anchored, Input, PatternSet,
|
||||
},
|
||||
regex_test::{
|
||||
CompiledRegex, Match, RegexTest, SearchKind, Span, TestResult,
|
||||
TestRunner,
|
||||
},
|
||||
};
|
||||
|
||||
use crate::{create_input, suite, untestify_kind};
|
||||
|
||||
const EXPANSIONS: &[&str] = &["is_match", "find", "which"];
|
||||
|
||||
/// Runs the test suite with the default configuration.
|
||||
#[test]
|
||||
fn unminimized_default() -> Result<()> {
|
||||
let builder = Regex::builder();
|
||||
TestRunner::new()?
|
||||
.expand(EXPANSIONS, |t| t.compiles())
|
||||
.blacklist("expensive")
|
||||
.test_iter(suite()?.iter(), dense_compiler(builder))
|
||||
.assert();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Runs the test suite with the default configuration and a prefilter enabled,
|
||||
/// if one can be built.
|
||||
#[test]
|
||||
fn unminimized_prefilter() -> Result<()> {
|
||||
let my_compiler = |test: &RegexTest, regexes: &[String]| {
|
||||
// Parse regexes as HIRs so we can get literals to build a prefilter.
|
||||
let mut hirs = vec![];
|
||||
for pattern in regexes.iter() {
|
||||
hirs.push(syntax::parse_with(pattern, &config_syntax(test))?);
|
||||
}
|
||||
let kind = match untestify_kind(test.match_kind()) {
|
||||
None => return Ok(CompiledRegex::skip()),
|
||||
Some(kind) => kind,
|
||||
};
|
||||
let pre = Prefilter::from_hirs_prefix(kind, &hirs);
|
||||
let mut builder = Regex::builder();
|
||||
builder.dense(dense::DFA::config().prefilter(pre));
|
||||
compiler(builder, |_, _, re| {
|
||||
Ok(CompiledRegex::compiled(move |test| -> TestResult {
|
||||
run_test(&re, test)
|
||||
}))
|
||||
})(test, regexes)
|
||||
};
|
||||
TestRunner::new()?
|
||||
.expand(EXPANSIONS, |t| t.compiles())
|
||||
.blacklist("expensive")
|
||||
.test_iter(suite()?.iter(), my_compiler)
|
||||
.assert();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Runs the test suite with start states specialized.
|
||||
#[test]
|
||||
fn unminimized_specialized_start_states() -> Result<()> {
|
||||
let mut builder = Regex::builder();
|
||||
builder.dense(dense::Config::new().specialize_start_states(true));
|
||||
|
||||
TestRunner::new()?
|
||||
.expand(EXPANSIONS, |t| t.compiles())
|
||||
.blacklist("expensive")
|
||||
.test_iter(suite()?.iter(), dense_compiler(builder))
|
||||
.assert();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Runs the test suite with byte classes disabled.
|
||||
#[test]
|
||||
fn unminimized_no_byte_class() -> Result<()> {
|
||||
let mut builder = Regex::builder();
|
||||
builder.dense(dense::Config::new().byte_classes(false));
|
||||
|
||||
TestRunner::new()?
|
||||
.expand(EXPANSIONS, |t| t.compiles())
|
||||
.blacklist("expensive")
|
||||
.test_iter(suite()?.iter(), dense_compiler(builder))
|
||||
.assert();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Runs the test suite with NFA shrinking enabled.
|
||||
#[test]
|
||||
fn unminimized_nfa_shrink() -> Result<()> {
|
||||
let mut builder = Regex::builder();
|
||||
builder.thompson(thompson::Config::new().shrink(true));
|
||||
|
||||
TestRunner::new()?
|
||||
.expand(EXPANSIONS, |t| t.compiles())
|
||||
.blacklist("expensive")
|
||||
.test_iter(suite()?.iter(), dense_compiler(builder))
|
||||
.assert();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Runs the test suite on a minimized DFA with an otherwise default
|
||||
/// configuration.
|
||||
#[test]
|
||||
fn minimized_default() -> Result<()> {
|
||||
let mut builder = Regex::builder();
|
||||
builder.dense(dense::Config::new().minimize(true));
|
||||
TestRunner::new()?
|
||||
.expand(EXPANSIONS, |t| t.compiles())
|
||||
.blacklist("expensive")
|
||||
.test_iter(suite()?.iter(), dense_compiler(builder))
|
||||
.assert();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Runs the test suite on a minimized DFA with byte classes disabled.
|
||||
#[test]
|
||||
fn minimized_no_byte_class() -> Result<()> {
|
||||
let mut builder = Regex::builder();
|
||||
builder.dense(dense::Config::new().minimize(true).byte_classes(false));
|
||||
|
||||
TestRunner::new()?
|
||||
.expand(EXPANSIONS, |t| t.compiles())
|
||||
.blacklist("expensive")
|
||||
.test_iter(suite()?.iter(), dense_compiler(builder))
|
||||
.assert();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Runs the test suite on a sparse unminimized DFA.
|
||||
#[test]
|
||||
fn sparse_unminimized_default() -> Result<()> {
|
||||
let builder = Regex::builder();
|
||||
TestRunner::new()?
|
||||
.expand(EXPANSIONS, |t| t.compiles())
|
||||
.blacklist("expensive")
|
||||
.test_iter(suite()?.iter(), sparse_compiler(builder))
|
||||
.assert();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Runs the test suite on a sparse unminimized DFA with prefilters enabled.
|
||||
#[test]
|
||||
fn sparse_unminimized_prefilter() -> Result<()> {
|
||||
let my_compiler = |test: &RegexTest, regexes: &[String]| {
|
||||
// Parse regexes as HIRs so we can get literals to build a prefilter.
|
||||
let mut hirs = vec![];
|
||||
for pattern in regexes.iter() {
|
||||
hirs.push(syntax::parse_with(pattern, &config_syntax(test))?);
|
||||
}
|
||||
let kind = match untestify_kind(test.match_kind()) {
|
||||
None => return Ok(CompiledRegex::skip()),
|
||||
Some(kind) => kind,
|
||||
};
|
||||
let pre = Prefilter::from_hirs_prefix(kind, &hirs);
|
||||
let mut builder = Regex::builder();
|
||||
builder.dense(dense::DFA::config().prefilter(pre));
|
||||
compiler(builder, |builder, _, re| {
|
||||
let fwd = re.forward().to_sparse()?;
|
||||
let rev = re.reverse().to_sparse()?;
|
||||
let re = builder.build_from_dfas(fwd, rev);
|
||||
Ok(CompiledRegex::compiled(move |test| -> TestResult {
|
||||
run_test(&re, test)
|
||||
}))
|
||||
})(test, regexes)
|
||||
};
|
||||
TestRunner::new()?
|
||||
.expand(EXPANSIONS, |t| t.compiles())
|
||||
.blacklist("expensive")
|
||||
.test_iter(suite()?.iter(), my_compiler)
|
||||
.assert();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Another basic sanity test that checks we can serialize and then deserialize
|
||||
/// a regex, and that the resulting regex can be used for searching correctly.
|
||||
#[test]
|
||||
fn serialization_unminimized_default() -> Result<()> {
|
||||
let builder = Regex::builder();
|
||||
let my_compiler = |builder| {
|
||||
compiler(builder, |builder, _, re| {
|
||||
let builder = builder.clone();
|
||||
let (fwd_bytes, _) = re.forward().to_bytes_native_endian();
|
||||
let (rev_bytes, _) = re.reverse().to_bytes_native_endian();
|
||||
Ok(CompiledRegex::compiled(move |test| -> TestResult {
|
||||
let fwd: dense::DFA<&[u32]> =
|
||||
dense::DFA::from_bytes(&fwd_bytes).unwrap().0;
|
||||
let rev: dense::DFA<&[u32]> =
|
||||
dense::DFA::from_bytes(&rev_bytes).unwrap().0;
|
||||
let re = builder.build_from_dfas(fwd, rev);
|
||||
|
||||
run_test(&re, test)
|
||||
}))
|
||||
})
|
||||
};
|
||||
TestRunner::new()?
|
||||
.expand(EXPANSIONS, |t| t.compiles())
|
||||
.blacklist("expensive")
|
||||
.test_iter(suite()?.iter(), my_compiler(builder))
|
||||
.assert();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A basic sanity test that checks we can serialize and then deserialize a
|
||||
/// regex using sparse DFAs, and that the resulting regex can be used for
|
||||
/// searching correctly.
|
||||
#[test]
|
||||
fn sparse_serialization_unminimized_default() -> Result<()> {
|
||||
let builder = Regex::builder();
|
||||
let my_compiler = |builder| {
|
||||
compiler(builder, |builder, _, re| {
|
||||
let builder = builder.clone();
|
||||
let fwd_bytes = re.forward().to_sparse()?.to_bytes_native_endian();
|
||||
let rev_bytes = re.reverse().to_sparse()?.to_bytes_native_endian();
|
||||
Ok(CompiledRegex::compiled(move |test| -> TestResult {
|
||||
let fwd: sparse::DFA<&[u8]> =
|
||||
sparse::DFA::from_bytes(&fwd_bytes).unwrap().0;
|
||||
let rev: sparse::DFA<&[u8]> =
|
||||
sparse::DFA::from_bytes(&rev_bytes).unwrap().0;
|
||||
let re = builder.build_from_dfas(fwd, rev);
|
||||
run_test(&re, test)
|
||||
}))
|
||||
})
|
||||
};
|
||||
TestRunner::new()?
|
||||
.expand(EXPANSIONS, |t| t.compiles())
|
||||
.blacklist("expensive")
|
||||
.test_iter(suite()?.iter(), my_compiler(builder))
|
||||
.assert();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn dense_compiler(
|
||||
builder: dfa::regex::Builder,
|
||||
) -> impl FnMut(&RegexTest, &[String]) -> Result<CompiledRegex> {
|
||||
compiler(builder, |_, _, re| {
|
||||
Ok(CompiledRegex::compiled(move |test| -> TestResult {
|
||||
run_test(&re, test)
|
||||
}))
|
||||
})
|
||||
}
|
||||
|
||||
fn sparse_compiler(
|
||||
builder: dfa::regex::Builder,
|
||||
) -> impl FnMut(&RegexTest, &[String]) -> Result<CompiledRegex> {
|
||||
compiler(builder, |builder, _, re| {
|
||||
let fwd = re.forward().to_sparse()?;
|
||||
let rev = re.reverse().to_sparse()?;
|
||||
let re = builder.build_from_dfas(fwd, rev);
|
||||
Ok(CompiledRegex::compiled(move |test| -> TestResult {
|
||||
run_test(&re, test)
|
||||
}))
|
||||
})
|
||||
}
|
||||
|
||||
fn compiler(
|
||||
mut builder: dfa::regex::Builder,
|
||||
mut create_matcher: impl FnMut(
|
||||
&dfa::regex::Builder,
|
||||
Option<Prefilter>,
|
||||
Regex,
|
||||
) -> Result<CompiledRegex>,
|
||||
) -> impl FnMut(&RegexTest, &[String]) -> Result<CompiledRegex> {
|
||||
move |test, regexes| {
|
||||
// Parse regexes as HIRs for some analysis below.
|
||||
let mut hirs = vec![];
|
||||
for pattern in regexes.iter() {
|
||||
hirs.push(syntax::parse_with(pattern, &config_syntax(test))?);
|
||||
}
|
||||
|
||||
// Get a prefilter in case the test wants it.
|
||||
let kind = match untestify_kind(test.match_kind()) {
|
||||
None => return Ok(CompiledRegex::skip()),
|
||||
Some(kind) => kind,
|
||||
};
|
||||
let pre = Prefilter::from_hirs_prefix(kind, &hirs);
|
||||
|
||||
// Check if our regex contains things that aren't supported by DFAs.
|
||||
// That is, Unicode word boundaries when searching non-ASCII text.
|
||||
if !test.haystack().is_ascii() {
|
||||
for hir in hirs.iter() {
|
||||
if hir.properties().look_set().contains_word_unicode() {
|
||||
return Ok(CompiledRegex::skip());
|
||||
}
|
||||
}
|
||||
}
|
||||
if !configure_regex_builder(test, &mut builder) {
|
||||
return Ok(CompiledRegex::skip());
|
||||
}
|
||||
create_matcher(&builder, pre, builder.build_many(®exes)?)
|
||||
}
|
||||
}
|
||||
|
||||
fn run_test<A: Automaton>(re: &Regex<A>, test: &RegexTest) -> TestResult {
|
||||
let input = create_input(test);
|
||||
match test.additional_name() {
|
||||
"is_match" => TestResult::matched(re.is_match(input.earliest(true))),
|
||||
"find" => match test.search_kind() {
|
||||
SearchKind::Earliest | SearchKind::Leftmost => {
|
||||
let input =
|
||||
input.earliest(test.search_kind() == SearchKind::Earliest);
|
||||
TestResult::matches(
|
||||
re.find_iter(input)
|
||||
.take(test.match_limit().unwrap_or(std::usize::MAX))
|
||||
.map(|m| Match {
|
||||
id: m.pattern().as_usize(),
|
||||
span: Span { start: m.start(), end: m.end() },
|
||||
}),
|
||||
)
|
||||
}
|
||||
SearchKind::Overlapping => {
|
||||
try_search_overlapping(re, &input).unwrap()
|
||||
}
|
||||
},
|
||||
"which" => match test.search_kind() {
|
||||
SearchKind::Earliest | SearchKind::Leftmost => {
|
||||
// There are no "which" APIs for standard searches.
|
||||
TestResult::skip()
|
||||
}
|
||||
SearchKind::Overlapping => {
|
||||
let dfa = re.forward();
|
||||
let mut patset = PatternSet::new(dfa.pattern_len());
|
||||
dfa.try_which_overlapping_matches(&input, &mut patset)
|
||||
.unwrap();
|
||||
TestResult::which(patset.iter().map(|p| p.as_usize()))
|
||||
}
|
||||
},
|
||||
name => TestResult::fail(&format!("unrecognized test name: {name}")),
|
||||
}
|
||||
}
|
||||
|
||||
/// Configures the given regex builder with all relevant settings on the given
|
||||
/// regex test.
|
||||
///
|
||||
/// If the regex test has a setting that is unsupported, then this returns
|
||||
/// false (implying the test should be skipped).
|
||||
fn configure_regex_builder(
|
||||
test: &RegexTest,
|
||||
builder: &mut dfa::regex::Builder,
|
||||
) -> bool {
|
||||
let match_kind = match untestify_kind(test.match_kind()) {
|
||||
None => return false,
|
||||
Some(k) => k,
|
||||
};
|
||||
|
||||
let starts = if test.anchored() {
|
||||
StartKind::Anchored
|
||||
} else {
|
||||
StartKind::Unanchored
|
||||
};
|
||||
let mut dense_config = dense::Config::new()
|
||||
.start_kind(starts)
|
||||
.match_kind(match_kind)
|
||||
.unicode_word_boundary(true);
|
||||
// When doing an overlapping search, we might try to find the start of each
|
||||
// match with a custom search routine. In that case, we need to tell the
|
||||
// reverse search (for the start offset) which pattern to look for. The
|
||||
// only way that API works is when anchored starting states are compiled
|
||||
// for each pattern. This does technically also enable it for the forward
|
||||
// DFA, but we're okay with that.
|
||||
if test.search_kind() == SearchKind::Overlapping {
|
||||
dense_config = dense_config.starts_for_each_pattern(true);
|
||||
}
|
||||
|
||||
builder
|
||||
.syntax(config_syntax(test))
|
||||
.thompson(config_thompson(test))
|
||||
.dense(dense_config);
|
||||
true
|
||||
}
|
||||
|
||||
/// Configuration of a Thompson NFA compiler from a regex test.
|
||||
fn config_thompson(test: &RegexTest) -> thompson::Config {
|
||||
let mut lookm = regex_automata::util::look::LookMatcher::new();
|
||||
lookm.set_line_terminator(test.line_terminator());
|
||||
thompson::Config::new().utf8(test.utf8()).look_matcher(lookm)
|
||||
}
|
||||
|
||||
/// Configuration of the regex syntax from a regex test.
|
||||
fn config_syntax(test: &RegexTest) -> syntax::Config {
|
||||
syntax::Config::new()
|
||||
.case_insensitive(test.case_insensitive())
|
||||
.unicode(test.unicode())
|
||||
.utf8(test.utf8())
|
||||
.line_terminator(test.line_terminator())
|
||||
}
|
||||
|
||||
/// Execute an overlapping search, and for each match found, also find its
|
||||
/// overlapping starting positions.
|
||||
///
|
||||
/// N.B. This routine used to be part of the crate API, but 1) it wasn't clear
|
||||
/// to me how useful it was and 2) it wasn't clear to me what its semantics
|
||||
/// should be. In particular, a potentially surprising footgun of this routine
|
||||
/// that it is worst case *quadratic* in the size of the haystack. Namely, it's
|
||||
/// possible to report a match at every position, and for every such position,
|
||||
/// scan all the way to the beginning of the haystack to find the starting
|
||||
/// position. Typical leftmost non-overlapping searches don't suffer from this
|
||||
/// because, well, matches can't overlap. So subsequent searches after a match
|
||||
/// is found don't revisit previously scanned parts of the haystack.
|
||||
///
|
||||
/// Its semantics can be strange for other reasons too. For example, given
|
||||
/// the regex '.*' and the haystack 'zz', the full set of overlapping matches
|
||||
/// is: [0, 0], [1, 1], [0, 1], [2, 2], [1, 2], [0, 2]. The ordering of
|
||||
/// those matches is quite strange, but makes sense when you think about the
|
||||
/// implementation: an end offset is found left-to-right, and then one or more
|
||||
/// starting offsets are found right-to-left.
|
||||
///
|
||||
/// Nevertheless, we provide this routine in our test suite because it's
|
||||
/// useful to test the low level DFA overlapping search and our test suite
|
||||
/// is written in a way that requires starting offsets.
|
||||
fn try_search_overlapping<A: Automaton>(
|
||||
re: &Regex<A>,
|
||||
input: &Input<'_>,
|
||||
) -> Result<TestResult> {
|
||||
let mut matches = vec![];
|
||||
let mut fwd_state = OverlappingState::start();
|
||||
let (fwd_dfa, rev_dfa) = (re.forward(), re.reverse());
|
||||
while let Some(end) = {
|
||||
fwd_dfa.try_search_overlapping_fwd(input, &mut fwd_state)?;
|
||||
fwd_state.get_match()
|
||||
} {
|
||||
let revsearch = input
|
||||
.clone()
|
||||
.range(input.start()..end.offset())
|
||||
.anchored(Anchored::Pattern(end.pattern()))
|
||||
.earliest(false);
|
||||
let mut rev_state = OverlappingState::start();
|
||||
while let Some(start) = {
|
||||
rev_dfa.try_search_overlapping_rev(&revsearch, &mut rev_state)?;
|
||||
rev_state.get_match()
|
||||
} {
|
||||
let span = Span { start: start.offset(), end: end.offset() };
|
||||
let mat = Match { id: end.pattern().as_usize(), span };
|
||||
matches.push(mat);
|
||||
}
|
||||
}
|
||||
Ok(TestResult::matches(matches))
|
||||
}
|
||||
Reference in new issue
Block a user