build(rust): vendor googletest crate
This commit was merged in pull request #76.
This commit is contained in:
alexion committed
1 parent
a6fe84190e
commit
6887b104d6
576 files changed
+284290
-2
No files matched your search
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,241 @@
|
||||
use crate::{hybrid::id::LazyStateIDError, nfa, util::search::Anchored};
|
||||
|
||||
/// An error that occurs when initial construction of a lazy DFA fails.
|
||||
///
|
||||
/// A build error can occur when insufficient cache capacity is configured or
|
||||
/// if something about the NFA is unsupported. (For example, if one attempts
|
||||
/// to build a lazy DFA without heuristic Unicode support but with an NFA that
|
||||
/// contains a Unicode word boundary.)
|
||||
///
|
||||
/// This error does not provide many introspection capabilities. There are
|
||||
/// generally only two things you can do with it:
|
||||
///
|
||||
/// * Obtain a human readable message via its `std::fmt::Display` impl.
|
||||
/// * Access an underlying
|
||||
/// [`nfa::thompson::BuildError`](crate::nfa::thompson::BuildError)
|
||||
/// type from its `source` method via the `std::error::Error` trait. This error
|
||||
/// only occurs when using convenience routines for building a lazy DFA
|
||||
/// directly from a pattern string.
|
||||
///
|
||||
/// When the `std` feature is enabled, this implements the `std::error::Error`
|
||||
/// trait.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct BuildError {
|
||||
kind: BuildErrorKind,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
enum BuildErrorKind {
|
||||
NFA(nfa::thompson::BuildError),
|
||||
InsufficientCacheCapacity { minimum: usize, given: usize },
|
||||
InsufficientStateIDCapacity { err: LazyStateIDError },
|
||||
Unsupported(&'static str),
|
||||
}
|
||||
|
||||
impl BuildError {
|
||||
pub(crate) fn nfa(err: nfa::thompson::BuildError) -> BuildError {
|
||||
BuildError { kind: BuildErrorKind::NFA(err) }
|
||||
}
|
||||
|
||||
pub(crate) fn insufficient_cache_capacity(
|
||||
minimum: usize,
|
||||
given: usize,
|
||||
) -> BuildError {
|
||||
BuildError {
|
||||
kind: BuildErrorKind::InsufficientCacheCapacity { minimum, given },
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn insufficient_state_id_capacity(
|
||||
err: LazyStateIDError,
|
||||
) -> BuildError {
|
||||
BuildError {
|
||||
kind: BuildErrorKind::InsufficientStateIDCapacity { err },
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn unsupported_dfa_word_boundary_unicode() -> BuildError {
|
||||
let msg = "cannot build lazy DFAs for regexes with Unicode word \
|
||||
boundaries; switch to ASCII word boundaries, or \
|
||||
heuristically enable Unicode word boundaries or use a \
|
||||
different regex engine";
|
||||
BuildError { kind: BuildErrorKind::Unsupported(msg) }
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "std")]
|
||||
impl std::error::Error for BuildError {
|
||||
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
|
||||
match self.kind {
|
||||
BuildErrorKind::NFA(ref err) => Some(err),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl core::fmt::Display for BuildError {
|
||||
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
|
||||
match self.kind {
|
||||
BuildErrorKind::NFA(_) => write!(f, "error building NFA"),
|
||||
BuildErrorKind::InsufficientCacheCapacity { minimum, given } => {
|
||||
write!(
|
||||
f,
|
||||
"given cache capacity ({given}) is smaller than \
|
||||
minimum required ({minimum})",
|
||||
)
|
||||
}
|
||||
BuildErrorKind::InsufficientStateIDCapacity { ref err } => {
|
||||
err.fmt(f)
|
||||
}
|
||||
BuildErrorKind::Unsupported(ref msg) => {
|
||||
write!(f, "unsupported regex feature for DFAs: {msg}")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// An error that can occur when computing the start state for a search.
|
||||
///
|
||||
/// Computing a start state can fail for a few reasons, either
|
||||
/// based on incorrect configuration or even based on whether
|
||||
/// the look-behind byte triggers a quit state. Typically
|
||||
/// one does not need to handle this error if you're using
|
||||
/// [`DFA::start_state_forward`](crate::hybrid::dfa::DFA::start_state_forward)
|
||||
/// (or its reverse counterpart), as that routine automatically converts
|
||||
/// `StartError` to a [`MatchError`](crate::MatchError) for you.
|
||||
///
|
||||
/// This error may be returned by the
|
||||
/// [`DFA::start_state`](crate::hybrid::dfa::DFA::start_state) routine.
|
||||
///
|
||||
/// This error implements the `std::error::Error` trait when the `std` feature
|
||||
/// is enabled.
|
||||
///
|
||||
/// This error is marked as non-exhaustive. New variants may be added in a
|
||||
/// semver compatible release.
|
||||
#[non_exhaustive]
|
||||
#[derive(Clone, Debug)]
|
||||
pub enum StartError {
|
||||
/// An error that occurs when cache inefficiency has dropped below the
|
||||
/// configured heuristic thresholds.
|
||||
Cache {
|
||||
/// The underlying cache error that occurred.
|
||||
err: CacheError,
|
||||
},
|
||||
/// An error that occurs when a starting configuration's look-behind byte
|
||||
/// is in this DFA's quit set.
|
||||
Quit {
|
||||
/// The quit byte that was found.
|
||||
byte: u8,
|
||||
},
|
||||
/// An error that occurs when the caller requests an anchored mode that
|
||||
/// isn't supported by the DFA.
|
||||
UnsupportedAnchored {
|
||||
/// The anchored mode given that is unsupported.
|
||||
mode: Anchored,
|
||||
},
|
||||
}
|
||||
|
||||
impl StartError {
|
||||
pub(crate) fn cache(err: CacheError) -> StartError {
|
||||
StartError::Cache { err }
|
||||
}
|
||||
|
||||
pub(crate) fn quit(byte: u8) -> StartError {
|
||||
StartError::Quit { byte }
|
||||
}
|
||||
|
||||
pub(crate) fn unsupported_anchored(mode: Anchored) -> StartError {
|
||||
StartError::UnsupportedAnchored { mode }
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "std")]
|
||||
impl std::error::Error for StartError {
|
||||
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
|
||||
match *self {
|
||||
StartError::Cache { ref err } => Some(err),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl core::fmt::Display for StartError {
|
||||
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
|
||||
match *self {
|
||||
StartError::Cache { .. } => write!(
|
||||
f,
|
||||
"error computing start state because of cache inefficiency"
|
||||
),
|
||||
StartError::Quit { byte } => write!(
|
||||
f,
|
||||
"error computing start state because the look-behind byte \
|
||||
{:?} triggered a quit state",
|
||||
crate::util::escape::DebugByte(byte),
|
||||
),
|
||||
StartError::UnsupportedAnchored { mode: Anchored::Yes } => {
|
||||
write!(
|
||||
f,
|
||||
"error computing start state because \
|
||||
anchored searches are not supported or enabled"
|
||||
)
|
||||
}
|
||||
StartError::UnsupportedAnchored { mode: Anchored::No } => {
|
||||
write!(
|
||||
f,
|
||||
"error computing start state because \
|
||||
unanchored searches are not supported or enabled"
|
||||
)
|
||||
}
|
||||
StartError::UnsupportedAnchored {
|
||||
mode: Anchored::Pattern(pid),
|
||||
} => {
|
||||
write!(
|
||||
f,
|
||||
"error computing start state because \
|
||||
anchored searches for a specific pattern ({}) \
|
||||
are not supported or enabled",
|
||||
pid.as_usize(),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// An error that occurs when cache usage has become inefficient.
|
||||
///
|
||||
/// One of the weaknesses of a lazy DFA is that it may need to clear its
|
||||
/// cache repeatedly if it's not big enough. If this happens too much, then it
|
||||
/// can slow searching down significantly. A mitigation to this is to use
|
||||
/// heuristics to detect whether the cache is being used efficiently or not.
|
||||
/// If not, then a lazy DFA can return a `CacheError`.
|
||||
///
|
||||
/// The default configuration of a lazy DFA in this crate is
|
||||
/// set such that a `CacheError` will never occur. Instead,
|
||||
/// callers must opt into this behavior with settings like
|
||||
/// [`dfa::Config::minimum_cache_clear_count`](crate::hybrid::dfa::Config::minimum_cache_clear_count)
|
||||
/// and
|
||||
/// [`dfa::Config::minimum_bytes_per_state`](crate::hybrid::dfa::Config::minimum_bytes_per_state).
|
||||
///
|
||||
/// When the `std` feature is enabled, this implements the `std::error::Error`
|
||||
/// trait.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct CacheError(());
|
||||
|
||||
impl CacheError {
|
||||
pub(crate) fn too_many_cache_clears() -> CacheError {
|
||||
CacheError(())
|
||||
}
|
||||
|
||||
pub(crate) fn bad_efficiency() -> CacheError {
|
||||
CacheError(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "std")]
|
||||
impl std::error::Error for CacheError {}
|
||||
|
||||
impl core::fmt::Display for CacheError {
|
||||
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
|
||||
write!(f, "lazy DFA cache has been cleared too many times")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,354 @@
|
||||
/// A state identifier specifically tailored for lazy DFAs.
|
||||
///
|
||||
/// A lazy state ID logically represents a pointer to a DFA state. In practice,
|
||||
/// by limiting the number of DFA states it can address, it reserves some
|
||||
/// bits of its representation to encode some additional information. That
|
||||
/// additional information is called a "tag." That tag is used to record
|
||||
/// whether the state it points to is an unknown, dead, quit, start or match
|
||||
/// state.
|
||||
///
|
||||
/// When implementing a low level search routine with a lazy DFA, it is
|
||||
/// necessary to query the type of the current state to know what to do:
|
||||
///
|
||||
/// * **Unknown** - The state has not yet been computed. The
|
||||
/// parameters used to get this state ID must be re-passed to
|
||||
/// [`DFA::next_state`](crate::hybrid::dfa::DFA::next_state), which will never
|
||||
/// return an unknown state ID.
|
||||
/// * **Dead** - A dead state only has transitions to itself. It indicates that
|
||||
/// the search cannot do anything else and should stop with whatever result it
|
||||
/// has.
|
||||
/// * **Quit** - A quit state indicates that the automaton could not answer
|
||||
/// whether a match exists or not. Correct search implementations must return a
|
||||
/// [`MatchError::quit`](crate::MatchError::quit) when a DFA enters a quit
|
||||
/// state.
|
||||
/// * **Start** - A start state is a state in which a search can begin.
|
||||
/// Lazy DFAs usually have more than one start state. Branching on
|
||||
/// this isn't required for correctness, but a common optimization is
|
||||
/// to run a prefilter when a search enters a start state. Note that
|
||||
/// start states are *not* tagged automatically, and one must enable the
|
||||
/// [`Config::specialize_start_states`](crate::hybrid::dfa::Config::specialize_start_states)
|
||||
/// setting for start states to be tagged. The reason for this is
|
||||
/// that a DFA search loop is usually written to execute a prefilter once it
|
||||
/// enters a start state. But if there is no prefilter, this handling can be
|
||||
/// quite disastrous as the DFA may ping-pong between the special handling code
|
||||
/// and a possible optimized hot path for handling untagged states. When start
|
||||
/// states aren't specialized, then they are untagged and remain in the hot
|
||||
/// path.
|
||||
/// * **Match** - A match state indicates that a match has been found.
|
||||
/// Depending on the semantics of your search implementation, it may either
|
||||
/// continue until the end of the haystack or a dead state, or it might quit
|
||||
/// and return the match immediately.
|
||||
///
|
||||
/// As an optimization, the [`is_tagged`](LazyStateID::is_tagged) predicate
|
||||
/// can be used to determine if a tag exists at all. This is useful to avoid
|
||||
/// branching on all of the above types for every byte searched.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// This example shows how `LazyStateID` can be used to implement a correct
|
||||
/// search routine with minimal branching. In particular, this search routine
|
||||
/// implements "leftmost" matching, which means that it doesn't immediately
|
||||
/// stop once a match is found. Instead, it continues until it reaches a dead
|
||||
/// state.
|
||||
///
|
||||
/// Notice also how a correct search implementation deals with
|
||||
/// [`CacheError`](crate::hybrid::CacheError)s returned by some of
|
||||
/// the lazy DFA routines. When a `CacheError` occurs, it returns
|
||||
/// [`MatchError::gave_up`](crate::MatchError::gave_up).
|
||||
///
|
||||
/// ```
|
||||
/// use regex_automata::{
|
||||
/// hybrid::dfa::{Cache, DFA},
|
||||
/// HalfMatch, MatchError, Input,
|
||||
/// };
|
||||
///
|
||||
/// fn find_leftmost_first(
|
||||
/// dfa: &DFA,
|
||||
/// cache: &mut Cache,
|
||||
/// haystack: &[u8],
|
||||
/// ) -> Result<Option<HalfMatch>, MatchError> {
|
||||
/// // The start state is determined by inspecting the position and the
|
||||
/// // initial bytes of the haystack. Note that start states can never
|
||||
/// // be match states (since DFAs in this crate delay matches by 1
|
||||
/// // byte), so we don't need to check if the start state is a match.
|
||||
/// let mut sid = dfa.start_state_forward(
|
||||
/// cache,
|
||||
/// &Input::new(haystack),
|
||||
/// )?;
|
||||
/// let mut last_match = None;
|
||||
/// // Walk all the bytes in the haystack. We can quit early if we see
|
||||
/// // a dead or a quit state. The former means the automaton will
|
||||
/// // never transition to any other state. The latter means that the
|
||||
/// // automaton entered a condition in which its search failed.
|
||||
/// for (i, &b) in haystack.iter().enumerate() {
|
||||
/// sid = dfa
|
||||
/// .next_state(cache, sid, b)
|
||||
/// .map_err(|_| MatchError::gave_up(i))?;
|
||||
/// if sid.is_tagged() {
|
||||
/// if sid.is_match() {
|
||||
/// last_match = Some(HalfMatch::new(
|
||||
/// dfa.match_pattern(cache, sid, 0),
|
||||
/// i,
|
||||
/// ));
|
||||
/// } else if sid.is_dead() {
|
||||
/// return Ok(last_match);
|
||||
/// } else if sid.is_quit() {
|
||||
/// // It is possible to enter into a quit state after
|
||||
/// // observing a match has occurred. In that case, we
|
||||
/// // should return the match instead of an error.
|
||||
/// if last_match.is_some() {
|
||||
/// return Ok(last_match);
|
||||
/// }
|
||||
/// return Err(MatchError::quit(b, i));
|
||||
/// }
|
||||
/// // Implementors may also want to check for start states and
|
||||
/// // handle them differently for performance reasons. But it is
|
||||
/// // not necessary for correctness. Note that in order to check
|
||||
/// // for start states, you'll need to enable the
|
||||
/// // 'specialize_start_states' config knob, otherwise start
|
||||
/// // states will not be tagged.
|
||||
/// }
|
||||
/// }
|
||||
/// // Matches are always delayed by 1 byte, so we must explicitly walk
|
||||
/// // the special "EOI" transition at the end of the search.
|
||||
/// sid = dfa
|
||||
/// .next_eoi_state(cache, sid)
|
||||
/// .map_err(|_| MatchError::gave_up(haystack.len()))?;
|
||||
/// if sid.is_match() {
|
||||
/// last_match = Some(HalfMatch::new(
|
||||
/// dfa.match_pattern(cache, sid, 0),
|
||||
/// haystack.len(),
|
||||
/// ));
|
||||
/// }
|
||||
/// Ok(last_match)
|
||||
/// }
|
||||
///
|
||||
/// // We use a greedy '+' operator to show how the search doesn't just stop
|
||||
/// // once a match is detected. It continues extending the match. Using
|
||||
/// // '[a-z]+?' would also work as expected and stop the search early.
|
||||
/// // Greediness is built into the automaton.
|
||||
/// let dfa = DFA::new(r"[a-z]+")?;
|
||||
/// let mut cache = dfa.create_cache();
|
||||
/// let haystack = "123 foobar 4567".as_bytes();
|
||||
/// let mat = find_leftmost_first(&dfa, &mut cache, haystack)?.unwrap();
|
||||
/// assert_eq!(mat.pattern().as_usize(), 0);
|
||||
/// assert_eq!(mat.offset(), 10);
|
||||
///
|
||||
/// // Here's another example that tests our handling of the special
|
||||
/// // EOI transition. This will fail to find a match if we don't call
|
||||
/// // 'next_eoi_state' at the end of the search since the match isn't found
|
||||
/// // until the final byte in the haystack.
|
||||
/// let dfa = DFA::new(r"[0-9]{4}")?;
|
||||
/// let mut cache = dfa.create_cache();
|
||||
/// let haystack = "123 foobar 4567".as_bytes();
|
||||
/// let mat = find_leftmost_first(&dfa, &mut cache, haystack)?.unwrap();
|
||||
/// assert_eq!(mat.pattern().as_usize(), 0);
|
||||
/// assert_eq!(mat.offset(), 15);
|
||||
///
|
||||
/// // And note that our search implementation above automatically works
|
||||
/// // with multi-DFAs. Namely, `dfa.match_pattern(match_state, 0)` selects
|
||||
/// // the appropriate pattern ID for us.
|
||||
/// let dfa = DFA::new_many(&[r"[a-z]+", r"[0-9]+"])?;
|
||||
/// let mut cache = dfa.create_cache();
|
||||
/// let haystack = "123 foobar 4567".as_bytes();
|
||||
/// let mat = find_leftmost_first(&dfa, &mut cache, haystack)?.unwrap();
|
||||
/// assert_eq!(mat.pattern().as_usize(), 1);
|
||||
/// assert_eq!(mat.offset(), 3);
|
||||
/// let mat = find_leftmost_first(&dfa, &mut cache, &haystack[3..])?.unwrap();
|
||||
/// assert_eq!(mat.pattern().as_usize(), 0);
|
||||
/// assert_eq!(mat.offset(), 7);
|
||||
/// let mat = find_leftmost_first(&dfa, &mut cache, &haystack[10..])?.unwrap();
|
||||
/// assert_eq!(mat.pattern().as_usize(), 1);
|
||||
/// assert_eq!(mat.offset(), 5);
|
||||
///
|
||||
/// # Ok::<(), Box<dyn std::error::Error>>(())
|
||||
/// ```
|
||||
#[derive(
|
||||
Clone, Copy, Debug, Default, Eq, Hash, PartialEq, PartialOrd, Ord,
|
||||
)]
|
||||
pub struct LazyStateID(u32);
|
||||
|
||||
impl LazyStateID {
|
||||
#[cfg(any(target_pointer_width = "32", target_pointer_width = "64"))]
|
||||
const MAX_BIT: usize = 31;
|
||||
|
||||
#[cfg(target_pointer_width = "16")]
|
||||
const MAX_BIT: usize = 15;
|
||||
|
||||
const MASK_UNKNOWN: usize = 1 << (LazyStateID::MAX_BIT);
|
||||
const MASK_DEAD: usize = 1 << (LazyStateID::MAX_BIT - 1);
|
||||
const MASK_QUIT: usize = 1 << (LazyStateID::MAX_BIT - 2);
|
||||
const MASK_START: usize = 1 << (LazyStateID::MAX_BIT - 3);
|
||||
const MASK_MATCH: usize = 1 << (LazyStateID::MAX_BIT - 4);
|
||||
pub(crate) const MAX: usize = LazyStateID::MASK_MATCH - 1;
|
||||
|
||||
/// Create a new lazy state ID.
|
||||
///
|
||||
/// If the given identifier exceeds [`LazyStateID::MAX`], then this returns
|
||||
/// an error.
|
||||
#[inline]
|
||||
pub(crate) fn new(id: usize) -> Result<LazyStateID, LazyStateIDError> {
|
||||
if id > LazyStateID::MAX {
|
||||
let attempted = u64::try_from(id).unwrap();
|
||||
return Err(LazyStateIDError { attempted });
|
||||
}
|
||||
Ok(LazyStateID::new_unchecked(id))
|
||||
}
|
||||
|
||||
/// Create a new lazy state ID without checking whether the given value
|
||||
/// exceeds [`LazyStateID::MAX`].
|
||||
///
|
||||
/// While this is unchecked, providing an incorrect value must never
|
||||
/// sacrifice memory safety.
|
||||
#[inline]
|
||||
const fn new_unchecked(id: usize) -> LazyStateID {
|
||||
// FIXME: Use as_u32() once const functions in traits are stable.
|
||||
LazyStateID(id as u32)
|
||||
}
|
||||
|
||||
/// Return this lazy state ID as an untagged `usize`.
|
||||
///
|
||||
/// If this lazy state ID is tagged, then the usize returned is the state
|
||||
/// ID without the tag. If the ID was not tagged, then the usize returned
|
||||
/// is equivalent to the state ID.
|
||||
#[inline]
|
||||
pub(crate) fn as_usize_untagged(&self) -> usize {
|
||||
self.as_usize_unchecked() & LazyStateID::MAX
|
||||
}
|
||||
|
||||
/// Return this lazy state ID as its raw internal `usize` value, which may
|
||||
/// be tagged (and thus greater than LazyStateID::MAX).
|
||||
#[inline]
|
||||
pub(crate) const fn as_usize_unchecked(&self) -> usize {
|
||||
// FIXME: Use as_usize() once const functions in traits are stable.
|
||||
self.0 as usize
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) const fn to_unknown(&self) -> LazyStateID {
|
||||
LazyStateID::new_unchecked(
|
||||
self.as_usize_unchecked() | LazyStateID::MASK_UNKNOWN,
|
||||
)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) const fn to_dead(&self) -> LazyStateID {
|
||||
LazyStateID::new_unchecked(
|
||||
self.as_usize_unchecked() | LazyStateID::MASK_DEAD,
|
||||
)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) const fn to_quit(&self) -> LazyStateID {
|
||||
LazyStateID::new_unchecked(
|
||||
self.as_usize_unchecked() | LazyStateID::MASK_QUIT,
|
||||
)
|
||||
}
|
||||
|
||||
/// Return this lazy state ID as a state ID that is tagged as a start
|
||||
/// state.
|
||||
#[inline]
|
||||
pub(crate) const fn to_start(&self) -> LazyStateID {
|
||||
LazyStateID::new_unchecked(
|
||||
self.as_usize_unchecked() | LazyStateID::MASK_START,
|
||||
)
|
||||
}
|
||||
|
||||
/// Return this lazy state ID as a lazy state ID that is tagged as a match
|
||||
/// state.
|
||||
#[inline]
|
||||
pub(crate) const fn to_match(&self) -> LazyStateID {
|
||||
LazyStateID::new_unchecked(
|
||||
self.as_usize_unchecked() | LazyStateID::MASK_MATCH,
|
||||
)
|
||||
}
|
||||
|
||||
/// Return true if and only if this lazy state ID is tagged.
|
||||
///
|
||||
/// When a lazy state ID is tagged, then one can conclude that it is one
|
||||
/// of a match, start, dead, quit or unknown state.
|
||||
#[inline]
|
||||
pub const fn is_tagged(&self) -> bool {
|
||||
self.as_usize_unchecked() > LazyStateID::MAX
|
||||
}
|
||||
|
||||
/// Return true if and only if this represents a lazy state ID that is
|
||||
/// "unknown." That is, the state has not yet been created. When a caller
|
||||
/// sees this state ID, it generally means that a state has to be computed
|
||||
/// in order to proceed.
|
||||
#[inline]
|
||||
pub const fn is_unknown(&self) -> bool {
|
||||
self.as_usize_unchecked() & LazyStateID::MASK_UNKNOWN > 0
|
||||
}
|
||||
|
||||
/// Return true if and only if this represents a dead state. A dead state
|
||||
/// is a state that can never transition to any other state except the
|
||||
/// dead state. When a dead state is seen, it generally indicates that a
|
||||
/// search should stop.
|
||||
#[inline]
|
||||
pub const fn is_dead(&self) -> bool {
|
||||
self.as_usize_unchecked() & LazyStateID::MASK_DEAD > 0
|
||||
}
|
||||
|
||||
/// Return true if and only if this represents a quit state. A quit state
|
||||
/// is a state that is representationally equivalent to a dead state,
|
||||
/// except it indicates the automaton has reached a point at which it can
|
||||
/// no longer determine whether a match exists or not. In general, this
|
||||
/// indicates an error during search and the caller must either pass this
|
||||
/// error up or use a different search technique.
|
||||
#[inline]
|
||||
pub const fn is_quit(&self) -> bool {
|
||||
self.as_usize_unchecked() & LazyStateID::MASK_QUIT > 0
|
||||
}
|
||||
|
||||
/// Return true if and only if this lazy state ID has been tagged as a
|
||||
/// start state.
|
||||
///
|
||||
/// Note that if
|
||||
/// [`Config::specialize_start_states`](crate::hybrid::dfa::Config) is
|
||||
/// disabled (which is the default), then this will always return false
|
||||
/// since start states won't be tagged.
|
||||
#[inline]
|
||||
pub const fn is_start(&self) -> bool {
|
||||
self.as_usize_unchecked() & LazyStateID::MASK_START > 0
|
||||
}
|
||||
|
||||
/// Return true if and only if this lazy state ID has been tagged as a
|
||||
/// match state.
|
||||
#[inline]
|
||||
pub const fn is_match(&self) -> bool {
|
||||
self.as_usize_unchecked() & LazyStateID::MASK_MATCH > 0
|
||||
}
|
||||
}
|
||||
|
||||
/// This error occurs when a lazy state ID could not be constructed.
|
||||
///
|
||||
/// This occurs when given an integer exceeding the maximum lazy state ID
|
||||
/// value.
|
||||
///
|
||||
/// When the `std` feature is enabled, this implements the `Error` trait.
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub(crate) struct LazyStateIDError {
|
||||
attempted: u64,
|
||||
}
|
||||
|
||||
impl LazyStateIDError {
|
||||
/// Returns the value that failed to constructed a lazy state ID.
|
||||
pub(crate) fn attempted(&self) -> u64 {
|
||||
self.attempted
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "std")]
|
||||
impl std::error::Error for LazyStateIDError {}
|
||||
|
||||
impl core::fmt::Display for LazyStateIDError {
|
||||
fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {
|
||||
write!(
|
||||
f,
|
||||
"failed to create LazyStateID from {:?}, which exceeds {:?}",
|
||||
self.attempted(),
|
||||
LazyStateID::MAX,
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
/*!
|
||||
A module for building and searching with lazy deterministic finite automata
|
||||
(DFAs).
|
||||
|
||||
Like other modules in this crate, lazy DFAs support a rich regex syntax with
|
||||
Unicode features. The key feature of a lazy DFA is that it builds itself
|
||||
incrementally during search, and never uses more than a configured capacity of
|
||||
memory. Thus, when searching with a lazy DFA, one must supply a mutable "cache"
|
||||
in which the actual DFA's transition table is stored.
|
||||
|
||||
If you're looking for fully compiled DFAs, then please see the top-level
|
||||
[`dfa` module](crate::dfa).
|
||||
|
||||
# Overview
|
||||
|
||||
This section gives a brief overview of the primary types in this module:
|
||||
|
||||
* A [`regex::Regex`] provides a way to search for matches of a regular
|
||||
expression using lazy DFAs. This includes iterating over matches with both the
|
||||
start and end positions of each match.
|
||||
* A [`dfa::DFA`] provides direct low level access to a lazy DFA.
|
||||
|
||||
# Example: basic regex searching
|
||||
|
||||
This example shows how to compile a regex using the default configuration
|
||||
and then use it to find matches in a byte string:
|
||||
|
||||
```
|
||||
use regex_automata::{hybrid::regex::Regex, Match};
|
||||
|
||||
let re = Regex::new(r"[0-9]{4}-[0-9]{2}-[0-9]{2}")?;
|
||||
let mut cache = re.create_cache();
|
||||
|
||||
let haystack = "2018-12-24 2016-10-08";
|
||||
let matches: Vec<Match> = re.find_iter(&mut cache, haystack).collect();
|
||||
assert_eq!(matches, vec![
|
||||
Match::must(0, 0..10),
|
||||
Match::must(0, 11..21),
|
||||
]);
|
||||
# Ok::<(), Box<dyn std::error::Error>>(())
|
||||
```
|
||||
|
||||
# Example: searching with multiple regexes
|
||||
|
||||
The lazy DFAs in this module all fully support searching with multiple regexes
|
||||
simultaneously. You can use this support with standard leftmost-first style
|
||||
searching to find non-overlapping matches:
|
||||
|
||||
```
|
||||
# if cfg!(miri) { return Ok(()); } // miri takes too long
|
||||
use regex_automata::{hybrid::regex::Regex, Match};
|
||||
|
||||
let re = Regex::new_many(&[r"\w+", r"\S+"])?;
|
||||
let mut cache = re.create_cache();
|
||||
|
||||
let haystack = "@foo bar";
|
||||
let matches: Vec<Match> = re.find_iter(&mut cache, haystack).collect();
|
||||
assert_eq!(matches, vec![
|
||||
Match::must(1, 0..4),
|
||||
Match::must(0, 5..8),
|
||||
]);
|
||||
# Ok::<(), Box<dyn std::error::Error>>(())
|
||||
```
|
||||
|
||||
# When should I use this?
|
||||
|
||||
Generally speaking, if you can abide the use of mutable state during search,
|
||||
and you don't need things like capturing groups or Unicode word boundary
|
||||
support in non-ASCII text, then a lazy DFA is likely a robust choice with
|
||||
respect to both search speed and memory usage. Note however that its speed
|
||||
may be worse than a general purpose regex engine if you don't select a good
|
||||
[prefilter](crate::util::prefilter).
|
||||
|
||||
If you know ahead of time that your pattern would result in a very large DFA
|
||||
if it was fully compiled, it may be better to use an NFA simulation instead
|
||||
of a lazy DFA. Either that, or increase the cache capacity of your lazy DFA
|
||||
to something that is big enough to hold the state machine (likely through
|
||||
experimentation). The issue here is that if the cache is too small, then it
|
||||
could wind up being reset too frequently and this might decrease searching
|
||||
speed significantly.
|
||||
|
||||
# Differences with fully compiled DFAs
|
||||
|
||||
A [`hybrid::regex::Regex`](crate::hybrid::regex::Regex) and a
|
||||
[`dfa::regex::Regex`](crate::dfa::regex::Regex) both have the same capabilities
|
||||
(and similarly for their underlying DFAs), but they achieve them through
|
||||
different means. The main difference is that a hybrid or "lazy" regex builds
|
||||
its DFA lazily during search, where as a fully compiled regex will build its
|
||||
DFA at construction time. While building a DFA at search time might sound like
|
||||
it's slow, it tends to work out where most bytes seen during a search will
|
||||
reuse pre-built parts of the DFA and thus can be almost as fast as a fully
|
||||
compiled DFA. The main downside is that searching requires mutable space to
|
||||
store the DFA, and, in the worst case, a search can result in a new state being
|
||||
created for each byte seen, which would make searching quite a bit slower.
|
||||
|
||||
A fully compiled DFA never has to worry about searches being slower once
|
||||
it's built. (Aside from, say, the transition table being so large that it
|
||||
is subject to harsh CPU cache effects.) However, of course, building a full
|
||||
DFA can be quite time consuming and memory hungry. Particularly when large
|
||||
Unicode character classes are used, which tend to translate into very large
|
||||
DFAs.
|
||||
|
||||
A lazy DFA strikes a nice balance _in practice_, particularly in the
|
||||
presence of Unicode mode, by only building what is needed. It avoids the
|
||||
worst case exponential time complexity of DFA compilation by guaranteeing that
|
||||
it will only build at most one state per byte searched. While the worst
|
||||
case here can lead to a very high constant, it will never be exponential.
|
||||
|
||||
# Syntax
|
||||
|
||||
This module supports the same syntax as the `regex` crate, since they share the
|
||||
same parser. You can find an exhaustive list of supported syntax in the
|
||||
[documentation for the `regex` crate](https://docs.rs/regex/1/regex/#syntax).
|
||||
|
||||
There are two things that are not supported by the lazy DFAs in this module:
|
||||
|
||||
* Capturing groups. The DFAs (and [`Regex`](regex::Regex)es built on top
|
||||
of them) can only find the offsets of an entire match, but cannot resolve
|
||||
the offsets of each capturing group. This is because DFAs do not have the
|
||||
expressive power necessary. Note that it is okay to build a lazy DFA from an
|
||||
NFA that contains capture groups. The capture groups will simply be ignored.
|
||||
* Unicode word boundaries. These present particularly difficult challenges for
|
||||
DFA construction and would result in an explosion in the number of states.
|
||||
One can enable [`dfa::Config::unicode_word_boundary`] though, which provides
|
||||
heuristic support for Unicode word boundaries that only works on ASCII text.
|
||||
Otherwise, one can use `(?-u:\b)` for an ASCII word boundary, which will work
|
||||
on any input.
|
||||
|
||||
There are no plans to lift either of these limitations.
|
||||
|
||||
Note that these restrictions are identical to the restrictions on fully
|
||||
compiled DFAs.
|
||||
*/
|
||||
|
||||
pub use self::{
|
||||
error::{BuildError, CacheError, StartError},
|
||||
id::LazyStateID,
|
||||
};
|
||||
|
||||
pub mod dfa;
|
||||
mod error;
|
||||
mod id;
|
||||
pub mod regex;
|
||||
mod search;
|
||||
@@ -0,0 +1,895 @@
|
||||
/*!
|
||||
A lazy DFA backed `Regex`.
|
||||
|
||||
This module provides a [`Regex`] backed by a lazy DFA. A `Regex` implements
|
||||
convenience routines you might have come to expect, such as finding a match
|
||||
and iterating over all non-overlapping matches. This `Regex` type is limited
|
||||
in its capabilities to what a lazy DFA can provide. Therefore, APIs involving
|
||||
capturing groups, for example, are not provided.
|
||||
|
||||
Internally, a `Regex` is composed of two DFAs. One is a "forward" DFA that
|
||||
finds the end offset of a match, where as the other is a "reverse" DFA that
|
||||
find the start offset of a match.
|
||||
|
||||
See the [parent module](crate::hybrid) for examples.
|
||||
*/
|
||||
|
||||
use crate::{
|
||||
hybrid::{
|
||||
dfa::{self, DFA},
|
||||
error::BuildError,
|
||||
},
|
||||
nfa::thompson,
|
||||
util::{
|
||||
iter,
|
||||
search::{Anchored, Input, Match, MatchError, MatchKind},
|
||||
},
|
||||
};
|
||||
|
||||
/// A regular expression that uses hybrid NFA/DFAs (also called "lazy DFAs")
|
||||
/// for searching.
|
||||
///
|
||||
/// A regular expression is comprised of two lazy DFAs, a "forward" DFA and a
|
||||
/// "reverse" DFA. The forward DFA is responsible for detecting the end of
|
||||
/// a match while the reverse DFA is responsible for detecting the start
|
||||
/// of a match. Thus, in order to find the bounds of any given match, a
|
||||
/// forward search must first be run followed by a reverse search. A match
|
||||
/// found by the forward DFA guarantees that the reverse DFA will also find
|
||||
/// a match.
|
||||
///
|
||||
/// # Fallibility
|
||||
///
|
||||
/// Most of the search routines defined on this type will _panic_ when the
|
||||
/// underlying search fails. This might be because the DFA gave up because it
|
||||
/// saw a quit byte, whether configured explicitly or via heuristic Unicode
|
||||
/// word boundary support, although neither are enabled by default. It might
|
||||
/// also fail if the underlying DFA determines it isn't making effective use of
|
||||
/// the cache (which also never happens by default). Or it might fail because
|
||||
/// an invalid `Input` configuration is given, for example, with an unsupported
|
||||
/// [`Anchored`] mode.
|
||||
///
|
||||
/// If you need to handle these error cases instead of allowing them to trigger
|
||||
/// a panic, then the lower level [`Regex::try_search`] provides a fallible API
|
||||
/// that never panics.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// This example shows how to cause a search to terminate if it sees a
|
||||
/// `\n` byte, and handle the error returned. This could be useful if, for
|
||||
/// example, you wanted to prevent a user supplied pattern from matching
|
||||
/// across a line boundary.
|
||||
///
|
||||
/// ```
|
||||
/// # if cfg!(miri) { return Ok(()); } // miri takes too long
|
||||
/// use regex_automata::{hybrid::{dfa, regex::Regex}, Input, MatchError};
|
||||
///
|
||||
/// let re = Regex::builder()
|
||||
/// .dfa(dfa::Config::new().quit(b'\n', true))
|
||||
/// .build(r"foo\p{any}+bar")?;
|
||||
/// let mut cache = re.create_cache();
|
||||
///
|
||||
/// let input = Input::new("foo\nbar");
|
||||
/// // Normally this would produce a match, since \p{any} contains '\n'.
|
||||
/// // But since we instructed the automaton to enter a quit state if a
|
||||
/// // '\n' is observed, this produces a match error instead.
|
||||
/// let expected = MatchError::quit(b'\n', 3);
|
||||
/// let got = re.try_search(&mut cache, &input).unwrap_err();
|
||||
/// assert_eq!(expected, got);
|
||||
///
|
||||
/// # Ok::<(), Box<dyn std::error::Error>>(())
|
||||
/// ```
|
||||
#[derive(Debug)]
|
||||
pub struct Regex {
|
||||
/// The forward lazy DFA. This can only find the end of a match.
|
||||
forward: DFA,
|
||||
/// The reverse lazy DFA. This can only find the start of a match.
|
||||
///
|
||||
/// This is built with 'all' match semantics (instead of leftmost-first)
|
||||
/// so that it always finds the longest possible match (which corresponds
|
||||
/// to the leftmost starting position). It is also compiled as an anchored
|
||||
/// matcher and has 'starts_for_each_pattern' enabled. Including starting
|
||||
/// states for each pattern is necessary to ensure that we only look for
|
||||
/// matches of a pattern that matched in the forward direction. Otherwise,
|
||||
/// we might wind up finding the "leftmost" starting position of a totally
|
||||
/// different pattern!
|
||||
reverse: DFA,
|
||||
}
|
||||
|
||||
/// Convenience routines for regex and cache construction.
|
||||
impl Regex {
|
||||
/// Parse the given regular expression using the default configuration and
|
||||
/// return the corresponding regex.
|
||||
///
|
||||
/// If you want a non-default configuration, then use the [`Builder`] to
|
||||
/// set your own configuration.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```
|
||||
/// use regex_automata::{hybrid::regex::Regex, Match};
|
||||
///
|
||||
/// let re = Regex::new("foo[0-9]+bar")?;
|
||||
/// let mut cache = re.create_cache();
|
||||
/// assert_eq!(
|
||||
/// Some(Match::must(0, 3..14)),
|
||||
/// re.find(&mut cache, "zzzfoo12345barzzz"),
|
||||
/// );
|
||||
/// # Ok::<(), Box<dyn std::error::Error>>(())
|
||||
/// ```
|
||||
#[cfg(feature = "syntax")]
|
||||
pub fn new(pattern: &str) -> Result<Regex, BuildError> {
|
||||
Regex::builder().build(pattern)
|
||||
}
|
||||
|
||||
/// Like `new`, but parses multiple patterns into a single "multi regex."
|
||||
/// This similarly uses the default regex configuration.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```
|
||||
/// use regex_automata::{hybrid::regex::Regex, Match};
|
||||
///
|
||||
/// let re = Regex::new_many(&["[a-z]+", "[0-9]+"])?;
|
||||
/// let mut cache = re.create_cache();
|
||||
///
|
||||
/// let mut it = re.find_iter(&mut cache, "abc 1 foo 4567 0 quux");
|
||||
/// assert_eq!(Some(Match::must(0, 0..3)), it.next());
|
||||
/// assert_eq!(Some(Match::must(1, 4..5)), it.next());
|
||||
/// assert_eq!(Some(Match::must(0, 6..9)), it.next());
|
||||
/// assert_eq!(Some(Match::must(1, 10..14)), it.next());
|
||||
/// assert_eq!(Some(Match::must(1, 15..16)), it.next());
|
||||
/// assert_eq!(Some(Match::must(0, 17..21)), it.next());
|
||||
/// assert_eq!(None, it.next());
|
||||
/// # Ok::<(), Box<dyn std::error::Error>>(())
|
||||
/// ```
|
||||
#[cfg(feature = "syntax")]
|
||||
pub fn new_many<P: AsRef<str>>(
|
||||
patterns: &[P],
|
||||
) -> Result<Regex, BuildError> {
|
||||
Regex::builder().build_many(patterns)
|
||||
}
|
||||
|
||||
/// Return a builder for configuring the construction of a `Regex`.
|
||||
///
|
||||
/// This is a convenience routine to avoid needing to import the
|
||||
/// [`Builder`] type in common cases.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// This example shows how to use the builder to disable UTF-8 mode
|
||||
/// everywhere.
|
||||
///
|
||||
/// ```
|
||||
/// # if cfg!(miri) { return Ok(()); } // miri takes too long
|
||||
/// use regex_automata::{
|
||||
/// hybrid::regex::Regex, nfa::thompson, util::syntax, Match,
|
||||
/// };
|
||||
///
|
||||
/// let re = Regex::builder()
|
||||
/// .syntax(syntax::Config::new().utf8(false))
|
||||
/// .thompson(thompson::Config::new().utf8(false))
|
||||
/// .build(r"foo(?-u:[^b])ar.*")?;
|
||||
/// let mut cache = re.create_cache();
|
||||
///
|
||||
/// let haystack = b"\xFEfoo\xFFarzz\xE2\x98\xFF\n";
|
||||
/// let expected = Some(Match::must(0, 1..9));
|
||||
/// let got = re.find(&mut cache, haystack);
|
||||
/// assert_eq!(expected, got);
|
||||
///
|
||||
/// # Ok::<(), Box<dyn std::error::Error>>(())
|
||||
/// ```
|
||||
pub fn builder() -> Builder {
|
||||
Builder::new()
|
||||
}
|
||||
|
||||
/// Create a new cache for this `Regex`.
|
||||
///
|
||||
/// The cache returned should only be used for searches for this
|
||||
/// `Regex`. If you want to reuse the cache for another `Regex`, then
|
||||
/// you must call [`Cache::reset`] with that `Regex` (or, equivalently,
|
||||
/// [`Regex::reset_cache`]).
|
||||
pub fn create_cache(&self) -> Cache {
|
||||
Cache::new(self)
|
||||
}
|
||||
|
||||
/// Reset the given cache such that it can be used for searching with the
|
||||
/// this `Regex` (and only this `Regex`).
|
||||
///
|
||||
/// A cache reset permits reusing memory already allocated in this cache
|
||||
/// with a different `Regex`.
|
||||
///
|
||||
/// Resetting a cache sets its "clear count" to 0. This is relevant if the
|
||||
/// `Regex` has been configured to "give up" after it has cleared the cache
|
||||
/// a certain number of times.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// This shows how to re-purpose a cache for use with a different `Regex`.
|
||||
///
|
||||
/// ```
|
||||
/// # if cfg!(miri) { return Ok(()); } // miri takes too long
|
||||
/// use regex_automata::{hybrid::regex::Regex, Match};
|
||||
///
|
||||
/// let re1 = Regex::new(r"\w")?;
|
||||
/// let re2 = Regex::new(r"\W")?;
|
||||
///
|
||||
/// let mut cache = re1.create_cache();
|
||||
/// assert_eq!(
|
||||
/// Some(Match::must(0, 0..2)),
|
||||
/// re1.find(&mut cache, "Δ"),
|
||||
/// );
|
||||
///
|
||||
/// // Using 'cache' with re2 is not allowed. It may result in panics or
|
||||
/// // incorrect results. In order to re-purpose the cache, we must reset
|
||||
/// // it with the Regex we'd like to use it with.
|
||||
/// //
|
||||
/// // Similarly, after this reset, using the cache with 're1' is also not
|
||||
/// // allowed.
|
||||
/// re2.reset_cache(&mut cache);
|
||||
/// assert_eq!(
|
||||
/// Some(Match::must(0, 0..3)),
|
||||
/// re2.find(&mut cache, "☃"),
|
||||
/// );
|
||||
///
|
||||
/// # Ok::<(), Box<dyn std::error::Error>>(())
|
||||
/// ```
|
||||
pub fn reset_cache(&self, cache: &mut Cache) {
|
||||
self.forward().reset_cache(&mut cache.forward);
|
||||
self.reverse().reset_cache(&mut cache.reverse);
|
||||
}
|
||||
}
|
||||
|
||||
/// Standard infallible search routines for finding and iterating over matches.
|
||||
impl Regex {
|
||||
/// Returns true if and only if this regex matches the given haystack.
|
||||
///
|
||||
/// This routine may short circuit if it knows that scanning future input
|
||||
/// will never lead to a different result. In particular, if the underlying
|
||||
/// DFA enters a match state or a dead state, then this routine will return
|
||||
/// `true` or `false`, respectively, without inspecting any future input.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// This routine panics if the search could not complete. This can occur
|
||||
/// in a number of circumstances:
|
||||
///
|
||||
/// * The configuration of the lazy DFA may permit it to "quit" the search.
|
||||
/// For example, setting quit bytes or enabling heuristic support for
|
||||
/// Unicode word boundaries. The default configuration does not enable any
|
||||
/// option that could result in the lazy DFA quitting.
|
||||
/// * The configuration of the lazy DFA may also permit it to "give up"
|
||||
/// on a search if it makes ineffective use of its transition table
|
||||
/// cache. The default configuration does not enable this by default,
|
||||
/// although it is typically a good idea to.
|
||||
/// * When the provided `Input` configuration is not supported. For
|
||||
/// example, by providing an unsupported anchor mode.
|
||||
///
|
||||
/// When a search panics, callers cannot know whether a match exists or
|
||||
/// not.
|
||||
///
|
||||
/// Use [`Regex::try_search`] if you want to handle these error conditions.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```
|
||||
/// use regex_automata::hybrid::regex::Regex;
|
||||
///
|
||||
/// let re = Regex::new("foo[0-9]+bar")?;
|
||||
/// let mut cache = re.create_cache();
|
||||
///
|
||||
/// assert!(re.is_match(&mut cache, "foo12345bar"));
|
||||
/// assert!(!re.is_match(&mut cache, "foobar"));
|
||||
/// # Ok::<(), Box<dyn std::error::Error>>(())
|
||||
/// ```
|
||||
#[inline]
|
||||
pub fn is_match<'h, I: Into<Input<'h>>>(
|
||||
&self,
|
||||
cache: &mut Cache,
|
||||
input: I,
|
||||
) -> bool {
|
||||
// Not only can we do an "earliest" search, but we can avoid doing a
|
||||
// reverse scan too.
|
||||
self.forward()
|
||||
.try_search_fwd(&mut cache.forward, &input.into().earliest(true))
|
||||
.unwrap()
|
||||
.is_some()
|
||||
}
|
||||
|
||||
/// Returns the start and end offset of the leftmost match. If no match
|
||||
/// exists, then `None` is returned.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// This routine panics if the search could not complete. This can occur
|
||||
/// in a number of circumstances:
|
||||
///
|
||||
/// * The configuration of the lazy DFA may permit it to "quit" the search.
|
||||
/// For example, setting quit bytes or enabling heuristic support for
|
||||
/// Unicode word boundaries. The default configuration does not enable any
|
||||
/// option that could result in the lazy DFA quitting.
|
||||
/// * The configuration of the lazy DFA may also permit it to "give up"
|
||||
/// on a search if it makes ineffective use of its transition table
|
||||
/// cache. The default configuration does not enable this by default,
|
||||
/// although it is typically a good idea to.
|
||||
/// * When the provided `Input` configuration is not supported. For
|
||||
/// example, by providing an unsupported anchor mode.
|
||||
///
|
||||
/// When a search panics, callers cannot know whether a match exists or
|
||||
/// not.
|
||||
///
|
||||
/// Use [`Regex::try_search`] if you want to handle these error conditions.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```
|
||||
/// use regex_automata::{Match, hybrid::regex::Regex};
|
||||
///
|
||||
/// let re = Regex::new("foo[0-9]+")?;
|
||||
/// let mut cache = re.create_cache();
|
||||
/// assert_eq!(
|
||||
/// Some(Match::must(0, 3..11)),
|
||||
/// re.find(&mut cache, "zzzfoo12345zzz"),
|
||||
/// );
|
||||
///
|
||||
/// // Even though a match is found after reading the first byte (`a`),
|
||||
/// // the default leftmost-first match semantics demand that we find the
|
||||
/// // earliest match that prefers earlier parts of the pattern over latter
|
||||
/// // parts.
|
||||
/// let re = Regex::new("abc|a")?;
|
||||
/// let mut cache = re.create_cache();
|
||||
/// assert_eq!(Some(Match::must(0, 0..3)), re.find(&mut cache, "abc"));
|
||||
/// # Ok::<(), Box<dyn std::error::Error>>(())
|
||||
/// ```
|
||||
#[inline]
|
||||
pub fn find<'h, I: Into<Input<'h>>>(
|
||||
&self,
|
||||
cache: &mut Cache,
|
||||
input: I,
|
||||
) -> Option<Match> {
|
||||
self.try_search(cache, &input.into()).unwrap()
|
||||
}
|
||||
|
||||
/// Returns an iterator over all non-overlapping leftmost matches in the
|
||||
/// given bytes. If no match exists, then the iterator yields no elements.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// This routine panics if the search could not complete. This can occur
|
||||
/// in a number of circumstances:
|
||||
///
|
||||
/// * The configuration of the lazy DFA may permit it to "quit" the search.
|
||||
/// For example, setting quit bytes or enabling heuristic support for
|
||||
/// Unicode word boundaries. The default configuration does not enable any
|
||||
/// option that could result in the lazy DFA quitting.
|
||||
/// * The configuration of the lazy DFA may also permit it to "give up"
|
||||
/// on a search if it makes ineffective use of its transition table
|
||||
/// cache. The default configuration does not enable this by default,
|
||||
/// although it is typically a good idea to.
|
||||
/// * When the provided `Input` configuration is not supported. For
|
||||
/// example, by providing an unsupported anchor mode.
|
||||
///
|
||||
/// When a search panics, callers cannot know whether a match exists or
|
||||
/// not.
|
||||
///
|
||||
/// The above conditions also apply to the iterator returned as well. For
|
||||
/// example, if the lazy DFA gives up or quits during a search using this
|
||||
/// method, then a panic will occur during iteration.
|
||||
///
|
||||
/// Use [`Regex::try_search`] with [`util::iter::Searcher`](iter::Searcher)
|
||||
/// if you want to handle these error conditions.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```
|
||||
/// use regex_automata::{hybrid::regex::Regex, Match};
|
||||
///
|
||||
/// let re = Regex::new("foo[0-9]+")?;
|
||||
/// let mut cache = re.create_cache();
|
||||
///
|
||||
/// let text = "foo1 foo12 foo123";
|
||||
/// let matches: Vec<Match> = re.find_iter(&mut cache, text).collect();
|
||||
/// assert_eq!(matches, vec![
|
||||
/// Match::must(0, 0..4),
|
||||
/// Match::must(0, 5..10),
|
||||
/// Match::must(0, 11..17),
|
||||
/// ]);
|
||||
/// # Ok::<(), Box<dyn std::error::Error>>(())
|
||||
/// ```
|
||||
#[inline]
|
||||
pub fn find_iter<'r, 'c, 'h, I: Into<Input<'h>>>(
|
||||
&'r self,
|
||||
cache: &'c mut Cache,
|
||||
input: I,
|
||||
) -> FindMatches<'r, 'c, 'h> {
|
||||
let it = iter::Searcher::new(input.into());
|
||||
FindMatches { re: self, cache, it }
|
||||
}
|
||||
}
|
||||
|
||||
/// Lower level "search" primitives that accept a `&Input` for cheap reuse
|
||||
/// and return an error if one occurs instead of panicking.
|
||||
impl Regex {
|
||||
/// Returns the start and end offset of the leftmost match. If no match
|
||||
/// exists, then `None` is returned.
|
||||
///
|
||||
/// This is like [`Regex::find`] but with two differences:
|
||||
///
|
||||
/// 1. It is not generic over `Into<Input>` and instead accepts a
|
||||
/// `&Input`. This permits reusing the same `Input` for multiple searches
|
||||
/// without needing to create a new one. This _may_ help with latency.
|
||||
/// 2. It returns an error if the search could not complete where as
|
||||
/// [`Regex::find`] will panic.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// This routine errors if the search could not complete. This can occur
|
||||
/// in a number of circumstances:
|
||||
///
|
||||
/// * The configuration of the lazy DFA may permit it to "quit" the search.
|
||||
/// For example, setting quit bytes or enabling heuristic support for
|
||||
/// Unicode word boundaries. The default configuration does not enable any
|
||||
/// option that could result in the lazy DFA quitting.
|
||||
/// * The configuration of the lazy DFA may also permit it to "give up"
|
||||
/// on a search if it makes ineffective use of its transition table
|
||||
/// cache. The default configuration does not enable this by default,
|
||||
/// although it is typically a good idea to.
|
||||
/// * When the provided `Input` configuration is not supported. For
|
||||
/// example, by providing an unsupported anchor mode.
|
||||
///
|
||||
/// When a search returns an error, callers cannot know whether a match
|
||||
/// exists or not.
|
||||
#[inline]
|
||||
pub fn try_search(
|
||||
&self,
|
||||
cache: &mut Cache,
|
||||
input: &Input<'_>,
|
||||
) -> Result<Option<Match>, MatchError> {
|
||||
let (fcache, rcache) = (&mut cache.forward, &mut cache.reverse);
|
||||
let end = match self.forward().try_search_fwd(fcache, input)? {
|
||||
None => return Ok(None),
|
||||
Some(end) => end,
|
||||
};
|
||||
// This special cases an empty match at the beginning of the search. If
|
||||
// our end matches our start, then since a reverse DFA can't match past
|
||||
// the start, it must follow that our starting position is also our end
|
||||
// position. So short circuit and skip the reverse search.
|
||||
if input.start() == end.offset() {
|
||||
return Ok(Some(Match::new(
|
||||
end.pattern(),
|
||||
end.offset()..end.offset(),
|
||||
)));
|
||||
}
|
||||
// We can also skip the reverse search if we know our search was
|
||||
// anchored. This occurs either when the input config is anchored or
|
||||
// when we know the regex itself is anchored. In this case, we know the
|
||||
// start of the match, if one is found, must be the start of the
|
||||
// search.
|
||||
if self.is_anchored(input) {
|
||||
return Ok(Some(Match::new(
|
||||
end.pattern(),
|
||||
input.start()..end.offset(),
|
||||
)));
|
||||
}
|
||||
// N.B. I have tentatively convinced myself that it isn't necessary
|
||||
// to specify the specific pattern for the reverse search since the
|
||||
// reverse search will always find the same pattern to match as the
|
||||
// forward search. But I lack a rigorous proof. Why not just provide
|
||||
// the pattern anyway? Well, if it is needed, then leaving it out
|
||||
// gives us a chance to find a witness. (Also, if we don't need to
|
||||
// specify the pattern, then we don't need to build the reverse DFA
|
||||
// with 'starts_for_each_pattern' enabled. It doesn't matter too much
|
||||
// for the lazy DFA, but does make the overall DFA bigger.)
|
||||
//
|
||||
// We also need to be careful to disable 'earliest' for the reverse
|
||||
// search, since it could be enabled for the forward search. In the
|
||||
// reverse case, to satisfy "leftmost" criteria, we need to match as
|
||||
// much as we can. We also need to be careful to make the search
|
||||
// anchored. We don't want the reverse search to report any matches
|
||||
// other than the one beginning at the end of our forward search.
|
||||
let revsearch = input
|
||||
.clone()
|
||||
.span(input.start()..end.offset())
|
||||
.anchored(Anchored::Yes)
|
||||
.earliest(false);
|
||||
let start = self
|
||||
.reverse()
|
||||
.try_search_rev(rcache, &revsearch)?
|
||||
.expect("reverse search must match if forward search does");
|
||||
debug_assert_eq!(
|
||||
start.pattern(),
|
||||
end.pattern(),
|
||||
"forward and reverse search must match same pattern",
|
||||
);
|
||||
debug_assert!(start.offset() <= end.offset());
|
||||
Ok(Some(Match::new(end.pattern(), start.offset()..end.offset())))
|
||||
}
|
||||
|
||||
/// Returns true if either the given input specifies an anchored search
|
||||
/// or if the underlying NFA is always anchored.
|
||||
fn is_anchored(&self, input: &Input<'_>) -> bool {
|
||||
match input.get_anchored() {
|
||||
Anchored::No => {
|
||||
self.forward().get_nfa().is_always_start_anchored()
|
||||
}
|
||||
Anchored::Yes | Anchored::Pattern(_) => true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Non-search APIs for querying information about the regex and setting a
|
||||
/// prefilter.
|
||||
impl Regex {
|
||||
/// Return the underlying lazy DFA responsible for forward matching.
|
||||
///
|
||||
/// This is useful for accessing the underlying lazy DFA and using it
|
||||
/// directly if the situation calls for it.
|
||||
pub fn forward(&self) -> &DFA {
|
||||
&self.forward
|
||||
}
|
||||
|
||||
/// Return the underlying lazy DFA responsible for reverse matching.
|
||||
///
|
||||
/// This is useful for accessing the underlying lazy DFA and using it
|
||||
/// directly if the situation calls for it.
|
||||
pub fn reverse(&self) -> &DFA {
|
||||
&self.reverse
|
||||
}
|
||||
|
||||
/// Returns the total number of patterns matched by this regex.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```
|
||||
/// # if cfg!(miri) { return Ok(()); } // miri takes too long
|
||||
/// use regex_automata::hybrid::regex::Regex;
|
||||
///
|
||||
/// let re = Regex::new_many(&[r"[a-z]+", r"[0-9]+", r"\w+"])?;
|
||||
/// assert_eq!(3, re.pattern_len());
|
||||
/// # Ok::<(), Box<dyn std::error::Error>>(())
|
||||
/// ```
|
||||
pub fn pattern_len(&self) -> usize {
|
||||
assert_eq!(self.forward().pattern_len(), self.reverse().pattern_len());
|
||||
self.forward().pattern_len()
|
||||
}
|
||||
}
|
||||
|
||||
/// An iterator over all non-overlapping matches for an infallible search.
|
||||
///
|
||||
/// The iterator yields a [`Match`] value until no more matches could be found.
|
||||
/// If the underlying regex engine returns an error, then a panic occurs.
|
||||
///
|
||||
/// The lifetime parameters are as follows:
|
||||
///
|
||||
/// * `'r` represents the lifetime of the regex object.
|
||||
/// * `'h` represents the lifetime of the haystack being searched.
|
||||
/// * `'c` represents the lifetime of the regex cache.
|
||||
///
|
||||
/// This iterator can be created with the [`Regex::find_iter`] method.
|
||||
#[derive(Debug)]
|
||||
pub struct FindMatches<'r, 'c, 'h> {
|
||||
re: &'r Regex,
|
||||
cache: &'c mut Cache,
|
||||
it: iter::Searcher<'h>,
|
||||
}
|
||||
|
||||
impl<'r, 'c, 'h> Iterator for FindMatches<'r, 'c, 'h> {
|
||||
type Item = Match;
|
||||
|
||||
#[inline]
|
||||
fn next(&mut self) -> Option<Match> {
|
||||
let FindMatches { re, ref mut cache, ref mut it } = *self;
|
||||
it.advance(|input| re.try_search(cache, input))
|
||||
}
|
||||
}
|
||||
|
||||
/// A cache represents a partially computed forward and reverse DFA.
|
||||
///
|
||||
/// A cache is the key component that differentiates a classical DFA and a
|
||||
/// hybrid NFA/DFA (also called a "lazy DFA"). Where a classical DFA builds a
|
||||
/// complete transition table that can handle all possible inputs, a hybrid
|
||||
/// NFA/DFA starts with an empty transition table and builds only the parts
|
||||
/// required during search. The parts that are built are stored in a cache. For
|
||||
/// this reason, a cache is a required parameter for nearly every operation on
|
||||
/// a [`Regex`].
|
||||
///
|
||||
/// Caches can be created from their corresponding `Regex` via
|
||||
/// [`Regex::create_cache`]. A cache can only be used with either the `Regex`
|
||||
/// that created it, or the `Regex` that was most recently used to reset it
|
||||
/// with [`Cache::reset`]. Using a cache with any other `Regex` may result in
|
||||
/// panics or incorrect results.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Cache {
|
||||
forward: dfa::Cache,
|
||||
reverse: dfa::Cache,
|
||||
}
|
||||
|
||||
impl Cache {
|
||||
/// Create a new cache for the given `Regex`.
|
||||
///
|
||||
/// The cache returned should only be used for searches for the given
|
||||
/// `Regex`. If you want to reuse the cache for another `Regex`, then you
|
||||
/// must call [`Cache::reset`] with that `Regex`.
|
||||
pub fn new(re: &Regex) -> Cache {
|
||||
let forward = dfa::Cache::new(re.forward());
|
||||
let reverse = dfa::Cache::new(re.reverse());
|
||||
Cache { forward, reverse }
|
||||
}
|
||||
|
||||
/// Reset this cache such that it can be used for searching with the given
|
||||
/// `Regex` (and only that `Regex`).
|
||||
///
|
||||
/// A cache reset permits reusing memory already allocated in this cache
|
||||
/// with a different `Regex`.
|
||||
///
|
||||
/// Resetting a cache sets its "clear count" to 0. This is relevant if the
|
||||
/// `Regex` has been configured to "give up" after it has cleared the cache
|
||||
/// a certain number of times.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// This shows how to re-purpose a cache for use with a different `Regex`.
|
||||
///
|
||||
/// ```
|
||||
/// # if cfg!(miri) { return Ok(()); } // miri takes too long
|
||||
/// use regex_automata::{hybrid::regex::Regex, Match};
|
||||
///
|
||||
/// let re1 = Regex::new(r"\w")?;
|
||||
/// let re2 = Regex::new(r"\W")?;
|
||||
///
|
||||
/// let mut cache = re1.create_cache();
|
||||
/// assert_eq!(
|
||||
/// Some(Match::must(0, 0..2)),
|
||||
/// re1.find(&mut cache, "Δ"),
|
||||
/// );
|
||||
///
|
||||
/// // Using 'cache' with re2 is not allowed. It may result in panics or
|
||||
/// // incorrect results. In order to re-purpose the cache, we must reset
|
||||
/// // it with the Regex we'd like to use it with.
|
||||
/// //
|
||||
/// // Similarly, after this reset, using the cache with 're1' is also not
|
||||
/// // allowed.
|
||||
/// cache.reset(&re2);
|
||||
/// assert_eq!(
|
||||
/// Some(Match::must(0, 0..3)),
|
||||
/// re2.find(&mut cache, "☃"),
|
||||
/// );
|
||||
///
|
||||
/// # Ok::<(), Box<dyn std::error::Error>>(())
|
||||
/// ```
|
||||
pub fn reset(&mut self, re: &Regex) {
|
||||
self.forward.reset(re.forward());
|
||||
self.reverse.reset(re.reverse());
|
||||
}
|
||||
|
||||
/// Return a reference to the forward cache.
|
||||
pub fn forward(&mut self) -> &dfa::Cache {
|
||||
&self.forward
|
||||
}
|
||||
|
||||
/// Return a reference to the reverse cache.
|
||||
pub fn reverse(&mut self) -> &dfa::Cache {
|
||||
&self.reverse
|
||||
}
|
||||
|
||||
/// Return a mutable reference to the forward cache.
|
||||
///
|
||||
/// If you need mutable references to both the forward and reverse caches,
|
||||
/// then use [`Cache::as_parts_mut`].
|
||||
pub fn forward_mut(&mut self) -> &mut dfa::Cache {
|
||||
&mut self.forward
|
||||
}
|
||||
|
||||
/// Return a mutable reference to the reverse cache.
|
||||
///
|
||||
/// If you need mutable references to both the forward and reverse caches,
|
||||
/// then use [`Cache::as_parts_mut`].
|
||||
pub fn reverse_mut(&mut self) -> &mut dfa::Cache {
|
||||
&mut self.reverse
|
||||
}
|
||||
|
||||
/// Return references to the forward and reverse caches, respectively.
|
||||
pub fn as_parts(&self) -> (&dfa::Cache, &dfa::Cache) {
|
||||
(&self.forward, &self.reverse)
|
||||
}
|
||||
|
||||
/// Return mutable references to the forward and reverse caches,
|
||||
/// respectively.
|
||||
pub fn as_parts_mut(&mut self) -> (&mut dfa::Cache, &mut dfa::Cache) {
|
||||
(&mut self.forward, &mut self.reverse)
|
||||
}
|
||||
|
||||
/// Returns the heap memory usage, in bytes, as a sum of the forward and
|
||||
/// reverse lazy DFA caches.
|
||||
///
|
||||
/// This does **not** include the stack size used up by this cache. To
|
||||
/// compute that, use `std::mem::size_of::<Cache>()`.
|
||||
pub fn memory_usage(&self) -> usize {
|
||||
self.forward.memory_usage() + self.reverse.memory_usage()
|
||||
}
|
||||
}
|
||||
|
||||
/// A builder for a regex based on a hybrid NFA/DFA.
|
||||
///
|
||||
/// This builder permits configuring options for the syntax of a pattern, the
|
||||
/// NFA construction, the lazy DFA construction and finally the regex searching
|
||||
/// itself. This builder is different from a general purpose regex builder
|
||||
/// in that it permits fine grain configuration of the construction process.
|
||||
/// The trade off for this is complexity, and the possibility of setting a
|
||||
/// configuration that might not make sense. For example, there are two
|
||||
/// different UTF-8 modes:
|
||||
///
|
||||
/// * [`syntax::Config::utf8`](crate::util::syntax::Config::utf8) controls
|
||||
/// whether the pattern itself can contain sub-expressions that match invalid
|
||||
/// UTF-8.
|
||||
/// * [`thompson::Config::utf8`] controls how the regex iterators themselves
|
||||
/// advance the starting position of the next search when a match with zero
|
||||
/// length is found.
|
||||
///
|
||||
/// Generally speaking, callers will want to either enable all of these or
|
||||
/// disable all of these.
|
||||
///
|
||||
/// Internally, building a regex requires building two hybrid NFA/DFAs,
|
||||
/// where one is responsible for finding the end of a match and the other is
|
||||
/// responsible for finding the start of a match. If you only need to detect
|
||||
/// whether something matched, or only the end of a match, then you should use
|
||||
/// a [`dfa::Builder`] to construct a single hybrid NFA/DFA, which is cheaper
|
||||
/// than building two of them.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// This example shows how to disable UTF-8 mode in the syntax and the regex
|
||||
/// itself. This is generally what you want for matching on arbitrary bytes.
|
||||
///
|
||||
/// ```
|
||||
/// # if cfg!(miri) { return Ok(()); } // miri takes too long
|
||||
/// use regex_automata::{
|
||||
/// hybrid::regex::Regex, nfa::thompson, util::syntax, Match,
|
||||
/// };
|
||||
///
|
||||
/// let re = Regex::builder()
|
||||
/// .syntax(syntax::Config::new().utf8(false))
|
||||
/// .thompson(thompson::Config::new().utf8(false))
|
||||
/// .build(r"foo(?-u:[^b])ar.*")?;
|
||||
/// let mut cache = re.create_cache();
|
||||
///
|
||||
/// let haystack = b"\xFEfoo\xFFarzz\xE2\x98\xFF\n";
|
||||
/// let expected = Some(Match::must(0, 1..9));
|
||||
/// let got = re.find(&mut cache, haystack);
|
||||
/// assert_eq!(expected, got);
|
||||
/// // Notice that `(?-u:[^b])` matches invalid UTF-8,
|
||||
/// // but the subsequent `.*` does not! Disabling UTF-8
|
||||
/// // on the syntax permits this.
|
||||
/// assert_eq!(b"foo\xFFarzz", &haystack[got.unwrap().range()]);
|
||||
///
|
||||
/// # Ok::<(), Box<dyn std::error::Error>>(())
|
||||
/// ```
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Builder {
|
||||
dfa: dfa::Builder,
|
||||
}
|
||||
|
||||
impl Builder {
|
||||
/// Create a new regex builder with the default configuration.
|
||||
pub fn new() -> Builder {
|
||||
Builder { dfa: DFA::builder() }
|
||||
}
|
||||
|
||||
/// Build a regex from the given pattern.
|
||||
///
|
||||
/// If there was a problem parsing or compiling the pattern, then an error
|
||||
/// is returned.
|
||||
#[cfg(feature = "syntax")]
|
||||
pub fn build(&self, pattern: &str) -> Result<Regex, BuildError> {
|
||||
self.build_many(&[pattern])
|
||||
}
|
||||
|
||||
/// Build a regex from the given patterns.
|
||||
#[cfg(feature = "syntax")]
|
||||
pub fn build_many<P: AsRef<str>>(
|
||||
&self,
|
||||
patterns: &[P],
|
||||
) -> Result<Regex, BuildError> {
|
||||
let forward = self.dfa.build_many(patterns)?;
|
||||
let reverse = self
|
||||
.dfa
|
||||
.clone()
|
||||
.configure(
|
||||
DFA::config()
|
||||
.prefilter(None)
|
||||
.specialize_start_states(false)
|
||||
.match_kind(MatchKind::All),
|
||||
)
|
||||
.thompson(thompson::Config::new().reverse(true))
|
||||
.build_many(patterns)?;
|
||||
Ok(self.build_from_dfas(forward, reverse))
|
||||
}
|
||||
|
||||
/// Build a regex from its component forward and reverse hybrid NFA/DFAs.
|
||||
///
|
||||
/// This is useful when you've built a forward and reverse lazy DFA
|
||||
/// separately, and want to combine them into a single regex. Once build,
|
||||
/// the individual DFAs given can still be accessed via [`Regex::forward`]
|
||||
/// and [`Regex::reverse`].
|
||||
///
|
||||
/// It is important that the reverse lazy DFA be compiled under the
|
||||
/// following conditions:
|
||||
///
|
||||
/// * It should use [`MatchKind::All`] semantics.
|
||||
/// * It should match in reverse.
|
||||
/// * Otherwise, its configuration should match the forward DFA.
|
||||
///
|
||||
/// If these conditions aren't satisfied, then the behavior of searches is
|
||||
/// unspecified.
|
||||
///
|
||||
/// Note that when using this constructor, no configuration is applied.
|
||||
/// Since this routine provides the DFAs to the builder, there is no
|
||||
/// opportunity to apply other configuration options.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// This shows how to build individual lazy forward and reverse DFAs, and
|
||||
/// then combine them into a single `Regex`.
|
||||
///
|
||||
/// ```
|
||||
/// use regex_automata::{
|
||||
/// hybrid::{dfa::DFA, regex::Regex},
|
||||
/// nfa::thompson,
|
||||
/// MatchKind,
|
||||
/// };
|
||||
///
|
||||
/// let fwd = DFA::new(r"foo[0-9]+")?;
|
||||
/// let rev = DFA::builder()
|
||||
/// .configure(DFA::config().match_kind(MatchKind::All))
|
||||
/// .thompson(thompson::Config::new().reverse(true))
|
||||
/// .build(r"foo[0-9]+")?;
|
||||
///
|
||||
/// let re = Regex::builder().build_from_dfas(fwd, rev);
|
||||
/// let mut cache = re.create_cache();
|
||||
/// assert_eq!(true, re.is_match(&mut cache, "foo123"));
|
||||
/// # Ok::<(), Box<dyn std::error::Error>>(())
|
||||
/// ```
|
||||
pub fn build_from_dfas(&self, forward: DFA, reverse: DFA) -> Regex {
|
||||
Regex { forward, reverse }
|
||||
}
|
||||
|
||||
/// Set the syntax configuration for this builder using
|
||||
/// [`syntax::Config`](crate::util::syntax::Config).
|
||||
///
|
||||
/// This permits setting things like case insensitivity, Unicode and multi
|
||||
/// line mode.
|
||||
#[cfg(feature = "syntax")]
|
||||
pub fn syntax(
|
||||
&mut self,
|
||||
config: crate::util::syntax::Config,
|
||||
) -> &mut Builder {
|
||||
self.dfa.syntax(config);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the Thompson NFA configuration for this builder using
|
||||
/// [`nfa::thompson::Config`](thompson::Config).
|
||||
///
|
||||
/// This permits setting things like whether additional time should be
|
||||
/// spent shrinking the size of the NFA.
|
||||
#[cfg(feature = "syntax")]
|
||||
pub fn thompson(&mut self, config: thompson::Config) -> &mut Builder {
|
||||
self.dfa.thompson(config);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the lazy DFA compilation configuration for this builder using
|
||||
/// [`dfa::Config`].
|
||||
///
|
||||
/// This permits setting things like whether Unicode word boundaries should
|
||||
/// be heuristically supported or settings how the behavior of the cache.
|
||||
pub fn dfa(&mut self, config: dfa::Config) -> &mut Builder {
|
||||
self.dfa.configure(config);
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for Builder {
|
||||
fn default() -> Builder {
|
||||
Builder::new()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,808 @@
|
||||
use crate::{
|
||||
hybrid::{
|
||||
dfa::{Cache, OverlappingState, DFA},
|
||||
id::LazyStateID,
|
||||
},
|
||||
util::{
|
||||
prefilter::Prefilter,
|
||||
search::{HalfMatch, Input, MatchError, Span},
|
||||
},
|
||||
};
|
||||
|
||||
#[inline(never)]
|
||||
pub(crate) fn find_fwd(
|
||||
dfa: &DFA,
|
||||
cache: &mut Cache,
|
||||
input: &Input<'_>,
|
||||
) -> Result<Option<HalfMatch>, MatchError> {
|
||||
if input.is_done() {
|
||||
return Ok(None);
|
||||
}
|
||||
let pre = if input.get_anchored().is_anchored() {
|
||||
None
|
||||
} else {
|
||||
dfa.get_config().get_prefilter()
|
||||
};
|
||||
// So what we do here is specialize four different versions of 'find_fwd':
|
||||
// one for each of the combinations for 'has prefilter' and 'is earliest
|
||||
// search'. The reason for doing this is that both of these things require
|
||||
// branches and special handling in some code that can be very hot,
|
||||
// and shaving off as much as we can when we don't need it tends to be
|
||||
// beneficial in ad hoc benchmarks. To see these differences, you often
|
||||
// need a query with a high match count. In other words, specializing these
|
||||
// four routines *tends* to help latency more than throughput.
|
||||
if pre.is_some() {
|
||||
if input.get_earliest() {
|
||||
find_fwd_imp(dfa, cache, input, pre, true)
|
||||
} else {
|
||||
find_fwd_imp(dfa, cache, input, pre, false)
|
||||
}
|
||||
} else {
|
||||
if input.get_earliest() {
|
||||
find_fwd_imp(dfa, cache, input, None, true)
|
||||
} else {
|
||||
find_fwd_imp(dfa, cache, input, None, false)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg_attr(feature = "perf-inline", inline(always))]
|
||||
fn find_fwd_imp(
|
||||
dfa: &DFA,
|
||||
cache: &mut Cache,
|
||||
input: &Input<'_>,
|
||||
pre: Option<&'_ Prefilter>,
|
||||
earliest: bool,
|
||||
) -> Result<Option<HalfMatch>, MatchError> {
|
||||
// See 'prefilter_restart' docs for explanation.
|
||||
let universal_start = dfa.get_nfa().look_set_prefix_any().is_empty();
|
||||
let mut mat = None;
|
||||
let mut sid = init_fwd(dfa, cache, input)?;
|
||||
let mut at = input.start();
|
||||
// This could just be a closure, but then I think it would be unsound
|
||||
// because it would need to be safe to invoke. This way, the lack of safety
|
||||
// is clearer in the code below.
|
||||
macro_rules! next_unchecked {
|
||||
($sid:expr, $at:expr) => {{
|
||||
let byte = *input.haystack().get_unchecked($at);
|
||||
dfa.next_state_untagged_unchecked(cache, $sid, byte)
|
||||
}};
|
||||
}
|
||||
|
||||
if let Some(ref pre) = pre {
|
||||
let span = Span::from(at..input.end());
|
||||
match pre.find(input.haystack(), span) {
|
||||
None => return Ok(mat),
|
||||
Some(ref span) => {
|
||||
at = span.start;
|
||||
if !universal_start {
|
||||
sid = prefilter_restart(dfa, cache, &input, at)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
cache.search_start(at);
|
||||
while at < input.end() {
|
||||
if sid.is_tagged() {
|
||||
cache.search_update(at);
|
||||
sid = dfa
|
||||
.next_state(cache, sid, input.haystack()[at])
|
||||
.map_err(|_| gave_up(at))?;
|
||||
} else {
|
||||
// SAFETY: There are two safety invariants we need to uphold
|
||||
// here in the loops below: that 'sid' and 'prev_sid' are valid
|
||||
// state IDs for this DFA, and that 'at' is a valid index into
|
||||
// 'haystack'. For the former, we rely on the invariant that
|
||||
// next_state* and start_state_forward always returns a valid state
|
||||
// ID (given a valid state ID in the former case), and that we are
|
||||
// only at this place in the code if 'sid' is untagged. Moreover,
|
||||
// every call to next_state_untagged_unchecked below is guarded by
|
||||
// a check that sid is untagged. For the latter safety invariant,
|
||||
// we always guard unchecked access with a check that 'at' is less
|
||||
// than 'end', where 'end <= haystack.len()'. In the unrolled loop
|
||||
// below, we ensure that 'at' is always in bounds.
|
||||
//
|
||||
// PERF: For justification of omitting bounds checks, it gives us a
|
||||
// ~10% bump in search time. This was used for a benchmark:
|
||||
//
|
||||
// regex-cli find half hybrid -p '(?m)^.+$' -UBb bigfile
|
||||
//
|
||||
// PERF: For justification for the loop unrolling, we use a few
|
||||
// different tests:
|
||||
//
|
||||
// regex-cli find half hybrid -p '\w{50}' -UBb bigfile
|
||||
// regex-cli find half hybrid -p '(?m)^.+$' -UBb bigfile
|
||||
// regex-cli find half hybrid -p 'ZQZQZQZQ' -UBb bigfile
|
||||
//
|
||||
// And there are three different configurations:
|
||||
//
|
||||
// nounroll: this entire 'else' block vanishes and we just
|
||||
// always use 'dfa.next_state(..)'.
|
||||
// unroll1: just the outer loop below
|
||||
// unroll2: just the inner loop below
|
||||
// unroll3: both the outer and inner loops below
|
||||
//
|
||||
// This results in a matrix of timings for each of the above
|
||||
// regexes with each of the above unrolling configurations:
|
||||
//
|
||||
// '\w{50}' '(?m)^.+$' 'ZQZQZQZQ'
|
||||
// nounroll 1.51s 2.34s 1.51s
|
||||
// unroll1 1.53s 2.32s 1.56s
|
||||
// unroll2 2.22s 1.50s 0.61s
|
||||
// unroll3 1.67s 1.45s 0.61s
|
||||
//
|
||||
// Ideally we'd be able to find a configuration that yields the
|
||||
// best time for all regexes, but alas we settle for unroll3 that
|
||||
// gives us *almost* the best for '\w{50}' and the best for the
|
||||
// other two regexes.
|
||||
//
|
||||
// So what exactly is going on here? The first unrolling (grouping
|
||||
// together runs of untagged transitions) specifically targets
|
||||
// our choice of representation. The second unrolling (grouping
|
||||
// together runs of self-transitions) specifically targets a common
|
||||
// DFA topology. Let's dig in a little bit by looking at our
|
||||
// regexes:
|
||||
//
|
||||
// '\w{50}': This regex spends a lot of time outside of the DFA's
|
||||
// start state matching some part of the '\w' repetition. This
|
||||
// means that it's a bit of a worst case for loop unrolling that
|
||||
// targets self-transitions since the self-transitions in '\w{50}'
|
||||
// are not particularly active for this haystack. However, the
|
||||
// first unrolling (grouping together untagged transitions)
|
||||
// does apply quite well here since very few transitions hit
|
||||
// match/dead/quit/unknown states. It is however worth mentioning
|
||||
// that if start states are configured to be tagged (which you
|
||||
// typically want to do if you have a prefilter), then this regex
|
||||
// actually slows way down because it is constantly ping-ponging
|
||||
// out of the unrolled loop and into the handling of a tagged start
|
||||
// state below. But when start states aren't tagged, the unrolled
|
||||
// loop stays hot. (This is why it's imperative that start state
|
||||
// tagging be disabled when there isn't a prefilter!)
|
||||
//
|
||||
// '(?m)^.+$': There are two important aspects of this regex: 1)
|
||||
// on this haystack, its match count is very high, much higher
|
||||
// than the other two regex and 2) it spends the vast majority
|
||||
// of its time matching '.+'. Since Unicode mode is disabled,
|
||||
// this corresponds to repeatedly following self transitions for
|
||||
// the vast majority of the input. This does benefit from the
|
||||
// untagged unrolling since most of the transitions will be to
|
||||
// untagged states, but the untagged unrolling does more work than
|
||||
// what is actually required. Namely, it has to keep track of the
|
||||
// previous and next state IDs, which I guess requires a bit more
|
||||
// shuffling. This is supported by the fact that nounroll+unroll1
|
||||
// are both slower than unroll2+unroll3, where the latter has a
|
||||
// loop unrolling that specifically targets self-transitions.
|
||||
//
|
||||
// 'ZQZQZQZQ': This one is very similar to '(?m)^.+$' because it
|
||||
// spends the vast majority of its time in self-transitions for
|
||||
// the (implicit) unanchored prefix. The main difference with
|
||||
// '(?m)^.+$' is that it has a much lower match count. So there
|
||||
// isn't much time spent in the overhead of reporting matches. This
|
||||
// is the primary explainer in the perf difference here. We include
|
||||
// this regex and the former to make sure we have comparison points
|
||||
// with high and low match counts.
|
||||
//
|
||||
// NOTE: I used 'OpenSubtitles2018.raw.sample.en' for 'bigfile'.
|
||||
//
|
||||
// NOTE: In a follow-up, it turns out that the "inner" loop
|
||||
// mentioned above was a pretty big pessimization in some other
|
||||
// cases. Namely, it resulted in too much ping-ponging into and out
|
||||
// of the loop, which resulted in nearly ~2x regressions in search
|
||||
// time when compared to the originally lazy DFA in the regex crate.
|
||||
// So I've removed the second loop unrolling that targets the
|
||||
// self-transition case.
|
||||
let mut prev_sid = sid;
|
||||
while at < input.end() {
|
||||
prev_sid = unsafe { next_unchecked!(sid, at) };
|
||||
if prev_sid.is_tagged() || at + 3 >= input.end() {
|
||||
core::mem::swap(&mut prev_sid, &mut sid);
|
||||
break;
|
||||
}
|
||||
at += 1;
|
||||
|
||||
sid = unsafe { next_unchecked!(prev_sid, at) };
|
||||
if sid.is_tagged() {
|
||||
break;
|
||||
}
|
||||
at += 1;
|
||||
|
||||
prev_sid = unsafe { next_unchecked!(sid, at) };
|
||||
if prev_sid.is_tagged() {
|
||||
core::mem::swap(&mut prev_sid, &mut sid);
|
||||
break;
|
||||
}
|
||||
at += 1;
|
||||
|
||||
sid = unsafe { next_unchecked!(prev_sid, at) };
|
||||
if sid.is_tagged() {
|
||||
break;
|
||||
}
|
||||
at += 1;
|
||||
}
|
||||
// If we quit out of the code above with an unknown state ID at
|
||||
// any point, then we need to re-compute that transition using
|
||||
// 'next_state', which will do NFA powerset construction for us.
|
||||
if sid.is_unknown() {
|
||||
cache.search_update(at);
|
||||
sid = dfa
|
||||
.next_state(cache, prev_sid, input.haystack()[at])
|
||||
.map_err(|_| gave_up(at))?;
|
||||
}
|
||||
}
|
||||
if sid.is_tagged() {
|
||||
if sid.is_start() {
|
||||
// A prefilter finds possible starts of new matches. Once a
|
||||
// match is pending, the DFA may still be resolving its
|
||||
// priority. Skipping ahead here could discard that match and
|
||||
// replace it with a later one.
|
||||
if let Some(ref pre) = pre {
|
||||
if mat.is_none() {
|
||||
let span = Span::from(at..input.end());
|
||||
match pre.find(input.haystack(), span) {
|
||||
None => {
|
||||
cache.search_finish(span.end);
|
||||
return Ok(mat);
|
||||
}
|
||||
Some(ref span) => {
|
||||
// We want to skip any update to 'at' below
|
||||
// at the end of this iteration and just
|
||||
// jump immediately back to the next state
|
||||
// transition at the leading position of the
|
||||
// candidate match.
|
||||
//
|
||||
// ... but only if we actually made progress
|
||||
// with our prefilter, otherwise if the start
|
||||
// state has a self-loop, we can get stuck.
|
||||
if span.start > at {
|
||||
at = span.start;
|
||||
if !universal_start {
|
||||
sid = prefilter_restart(
|
||||
dfa, cache, &input, at,
|
||||
)?;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if sid.is_match() {
|
||||
let pattern = dfa.match_pattern(cache, sid, 0);
|
||||
// Since slice ranges are inclusive at the beginning and
|
||||
// exclusive at the end, and since forward searches report
|
||||
// the end, we can return 'at' as-is. This only works because
|
||||
// matches are delayed by 1 byte. So by the time we observe a
|
||||
// match, 'at' has already been set to 1 byte past the actual
|
||||
// match location, which is precisely the exclusive ending
|
||||
// bound of the match.
|
||||
mat = Some(HalfMatch::new(pattern, at));
|
||||
if earliest {
|
||||
cache.search_finish(at);
|
||||
return Ok(mat);
|
||||
}
|
||||
} else if sid.is_dead() {
|
||||
cache.search_finish(at);
|
||||
return Ok(mat);
|
||||
} else if sid.is_quit() {
|
||||
cache.search_finish(at);
|
||||
return Err(MatchError::quit(input.haystack()[at], at));
|
||||
} else {
|
||||
debug_assert!(sid.is_unknown());
|
||||
unreachable!("sid being unknown is a bug");
|
||||
}
|
||||
}
|
||||
at += 1;
|
||||
}
|
||||
eoi_fwd(dfa, cache, input, &mut sid, &mut mat)?;
|
||||
cache.search_finish(input.end());
|
||||
Ok(mat)
|
||||
}
|
||||
|
||||
#[inline(never)]
|
||||
pub(crate) fn find_rev(
|
||||
dfa: &DFA,
|
||||
cache: &mut Cache,
|
||||
input: &Input<'_>,
|
||||
) -> Result<Option<HalfMatch>, MatchError> {
|
||||
if input.is_done() {
|
||||
return Ok(None);
|
||||
}
|
||||
if input.get_earliest() {
|
||||
find_rev_imp(dfa, cache, input, true)
|
||||
} else {
|
||||
find_rev_imp(dfa, cache, input, false)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg_attr(feature = "perf-inline", inline(always))]
|
||||
fn find_rev_imp(
|
||||
dfa: &DFA,
|
||||
cache: &mut Cache,
|
||||
input: &Input<'_>,
|
||||
earliest: bool,
|
||||
) -> Result<Option<HalfMatch>, MatchError> {
|
||||
let mut mat = None;
|
||||
let mut sid = init_rev(dfa, cache, input)?;
|
||||
// In reverse search, the loop below can't handle the case of searching an
|
||||
// empty slice. Ideally we could write something congruent to the forward
|
||||
// search, i.e., 'while at >= start', but 'start' might be 0. Since we use
|
||||
// an unsigned offset, 'at >= 0' is trivially always true. We could avoid
|
||||
// this extra case handling by using a signed offset, but Rust makes it
|
||||
// annoying to do. So... We just handle the empty case separately.
|
||||
if input.start() == input.end() {
|
||||
eoi_rev(dfa, cache, input, &mut sid, &mut mat)?;
|
||||
return Ok(mat);
|
||||
}
|
||||
|
||||
let mut at = input.end() - 1;
|
||||
macro_rules! next_unchecked {
|
||||
($sid:expr, $at:expr) => {{
|
||||
let byte = *input.haystack().get_unchecked($at);
|
||||
dfa.next_state_untagged_unchecked(cache, $sid, byte)
|
||||
}};
|
||||
}
|
||||
cache.search_start(at);
|
||||
loop {
|
||||
if sid.is_tagged() {
|
||||
cache.search_update(at);
|
||||
sid = dfa
|
||||
.next_state(cache, sid, input.haystack()[at])
|
||||
.map_err(|_| gave_up(at))?;
|
||||
} else {
|
||||
// SAFETY: See comments in 'find_fwd' for a safety argument.
|
||||
//
|
||||
// PERF: The comments in 'find_fwd' also provide a justification
|
||||
// from a performance perspective as to 1) why we elide bounds
|
||||
// checks and 2) why we do a specialized version of unrolling
|
||||
// below. The reverse search does have a slightly different
|
||||
// consideration in that most reverse searches tend to be
|
||||
// anchored and on shorter haystacks. However, this still makes a
|
||||
// difference. Take this command for example:
|
||||
//
|
||||
// regex-cli find match hybrid -p '(?m)^.+$' -UBb bigfile
|
||||
//
|
||||
// (Notice that we use 'find hybrid regex', not 'find hybrid dfa'
|
||||
// like in the justification for the forward direction. The 'regex'
|
||||
// sub-command will find start-of-match and thus run the reverse
|
||||
// direction.)
|
||||
//
|
||||
// Without unrolling below, the above command takes around 3.76s.
|
||||
// But with the unrolling below, we get down to 2.55s. If we keep
|
||||
// the unrolling but add in bounds checks, then we get 2.86s.
|
||||
//
|
||||
// NOTE: I used 'OpenSubtitles2018.raw.sample.en' for 'bigfile'.
|
||||
let mut prev_sid = sid;
|
||||
while at >= input.start() {
|
||||
prev_sid = unsafe { next_unchecked!(sid, at) };
|
||||
if prev_sid.is_tagged()
|
||||
|| at <= input.start().saturating_add(3)
|
||||
{
|
||||
core::mem::swap(&mut prev_sid, &mut sid);
|
||||
break;
|
||||
}
|
||||
at -= 1;
|
||||
|
||||
sid = unsafe { next_unchecked!(prev_sid, at) };
|
||||
if sid.is_tagged() {
|
||||
break;
|
||||
}
|
||||
at -= 1;
|
||||
|
||||
prev_sid = unsafe { next_unchecked!(sid, at) };
|
||||
if prev_sid.is_tagged() {
|
||||
core::mem::swap(&mut prev_sid, &mut sid);
|
||||
break;
|
||||
}
|
||||
at -= 1;
|
||||
|
||||
sid = unsafe { next_unchecked!(prev_sid, at) };
|
||||
if sid.is_tagged() {
|
||||
break;
|
||||
}
|
||||
at -= 1;
|
||||
}
|
||||
// If we quit out of the code above with an unknown state ID at
|
||||
// any point, then we need to re-compute that transition using
|
||||
// 'next_state', which will do NFA powerset construction for us.
|
||||
if sid.is_unknown() {
|
||||
cache.search_update(at);
|
||||
sid = dfa
|
||||
.next_state(cache, prev_sid, input.haystack()[at])
|
||||
.map_err(|_| gave_up(at))?;
|
||||
}
|
||||
}
|
||||
if sid.is_tagged() {
|
||||
if sid.is_start() {
|
||||
// do nothing
|
||||
} else if sid.is_match() {
|
||||
let pattern = dfa.match_pattern(cache, sid, 0);
|
||||
// Since reverse searches report the beginning of a match
|
||||
// and the beginning is inclusive (not exclusive like the
|
||||
// end of a match), we add 1 to make it inclusive.
|
||||
mat = Some(HalfMatch::new(pattern, at + 1));
|
||||
if earliest {
|
||||
cache.search_finish(at);
|
||||
return Ok(mat);
|
||||
}
|
||||
} else if sid.is_dead() {
|
||||
cache.search_finish(at);
|
||||
return Ok(mat);
|
||||
} else if sid.is_quit() {
|
||||
cache.search_finish(at);
|
||||
return Err(MatchError::quit(input.haystack()[at], at));
|
||||
} else {
|
||||
debug_assert!(sid.is_unknown());
|
||||
unreachable!("sid being unknown is a bug");
|
||||
}
|
||||
}
|
||||
if at == input.start() {
|
||||
break;
|
||||
}
|
||||
at -= 1;
|
||||
}
|
||||
cache.search_finish(input.start());
|
||||
eoi_rev(dfa, cache, input, &mut sid, &mut mat)?;
|
||||
Ok(mat)
|
||||
}
|
||||
|
||||
#[inline(never)]
|
||||
pub(crate) fn find_overlapping_fwd(
|
||||
dfa: &DFA,
|
||||
cache: &mut Cache,
|
||||
input: &Input<'_>,
|
||||
state: &mut OverlappingState,
|
||||
) -> Result<(), MatchError> {
|
||||
state.mat = None;
|
||||
if input.is_done() {
|
||||
return Ok(());
|
||||
}
|
||||
let pre = if input.get_anchored().is_anchored() {
|
||||
None
|
||||
} else {
|
||||
dfa.get_config().get_prefilter()
|
||||
};
|
||||
if pre.is_some() {
|
||||
find_overlapping_fwd_imp(dfa, cache, input, pre, state)
|
||||
} else {
|
||||
find_overlapping_fwd_imp(dfa, cache, input, None, state)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg_attr(feature = "perf-inline", inline(always))]
|
||||
fn find_overlapping_fwd_imp(
|
||||
dfa: &DFA,
|
||||
cache: &mut Cache,
|
||||
input: &Input<'_>,
|
||||
pre: Option<&'_ Prefilter>,
|
||||
state: &mut OverlappingState,
|
||||
) -> Result<(), MatchError> {
|
||||
// See 'prefilter_restart' docs for explanation.
|
||||
let universal_start = dfa.get_nfa().look_set_prefix_any().is_empty();
|
||||
let mut sid = match state.id {
|
||||
None => {
|
||||
state.at = input.start();
|
||||
init_fwd(dfa, cache, input)?
|
||||
}
|
||||
Some(sid) => {
|
||||
if let Some(match_index) = state.next_match_index {
|
||||
let match_len = dfa.match_len(cache, sid);
|
||||
if match_index < match_len {
|
||||
state.next_match_index = Some(match_index + 1);
|
||||
let pattern = dfa.match_pattern(cache, sid, match_index);
|
||||
state.mat = Some(HalfMatch::new(pattern, state.at));
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
// Once we've reported all matches at a given position, we need to
|
||||
// advance the search to the next position.
|
||||
state.at += 1;
|
||||
if state.at > input.end() {
|
||||
return Ok(());
|
||||
}
|
||||
sid
|
||||
}
|
||||
};
|
||||
|
||||
// NOTE: We don't optimize the crap out of this routine primarily because
|
||||
// it seems like most overlapping searches will have higher match counts,
|
||||
// and thus, throughput is perhaps not as important. But if you have a use
|
||||
// case for something faster, feel free to file an issue.
|
||||
cache.search_start(state.at);
|
||||
while state.at < input.end() {
|
||||
sid = dfa
|
||||
.next_state(cache, sid, input.haystack()[state.at])
|
||||
.map_err(|_| gave_up(state.at))?;
|
||||
if sid.is_tagged() {
|
||||
state.id = Some(sid);
|
||||
if sid.is_start() {
|
||||
if let Some(ref pre) = pre {
|
||||
let span = Span::from(state.at..input.end());
|
||||
match pre.find(input.haystack(), span) {
|
||||
None => return Ok(()),
|
||||
Some(ref span) => {
|
||||
if span.start > state.at {
|
||||
state.at = span.start;
|
||||
if !universal_start {
|
||||
sid = prefilter_restart(
|
||||
dfa, cache, &input, state.at,
|
||||
)?;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if sid.is_match() {
|
||||
state.next_match_index = Some(1);
|
||||
let pattern = dfa.match_pattern(cache, sid, 0);
|
||||
state.mat = Some(HalfMatch::new(pattern, state.at));
|
||||
cache.search_finish(state.at);
|
||||
return Ok(());
|
||||
} else if sid.is_dead() {
|
||||
cache.search_finish(state.at);
|
||||
return Ok(());
|
||||
} else if sid.is_quit() {
|
||||
cache.search_finish(state.at);
|
||||
return Err(MatchError::quit(
|
||||
input.haystack()[state.at],
|
||||
state.at,
|
||||
));
|
||||
} else {
|
||||
debug_assert!(sid.is_unknown());
|
||||
unreachable!("sid being unknown is a bug");
|
||||
}
|
||||
}
|
||||
state.at += 1;
|
||||
cache.search_update(state.at);
|
||||
}
|
||||
|
||||
let result = eoi_fwd(dfa, cache, input, &mut sid, &mut state.mat);
|
||||
state.id = Some(sid);
|
||||
if state.mat.is_some() {
|
||||
// '1' is always correct here since if we get to this point, this
|
||||
// always corresponds to the first (index '0') match discovered at
|
||||
// this position. So the next match to report at this position (if
|
||||
// it exists) is at index '1'.
|
||||
state.next_match_index = Some(1);
|
||||
}
|
||||
cache.search_finish(input.end());
|
||||
result
|
||||
}
|
||||
|
||||
#[inline(never)]
|
||||
pub(crate) fn find_overlapping_rev(
|
||||
dfa: &DFA,
|
||||
cache: &mut Cache,
|
||||
input: &Input<'_>,
|
||||
state: &mut OverlappingState,
|
||||
) -> Result<(), MatchError> {
|
||||
state.mat = None;
|
||||
if input.is_done() {
|
||||
return Ok(());
|
||||
}
|
||||
let mut sid = match state.id {
|
||||
None => {
|
||||
let sid = init_rev(dfa, cache, input)?;
|
||||
state.id = Some(sid);
|
||||
if input.start() == input.end() {
|
||||
state.rev_eoi = true;
|
||||
} else {
|
||||
state.at = input.end() - 1;
|
||||
}
|
||||
sid
|
||||
}
|
||||
Some(sid) => {
|
||||
if let Some(match_index) = state.next_match_index {
|
||||
let match_len = dfa.match_len(cache, sid);
|
||||
if match_index < match_len {
|
||||
state.next_match_index = Some(match_index + 1);
|
||||
let pattern = dfa.match_pattern(cache, sid, match_index);
|
||||
state.mat = Some(HalfMatch::new(pattern, state.at));
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
// Once we've reported all matches at a given position, we need
|
||||
// to advance the search to the next position. However, if we've
|
||||
// already followed the EOI transition, then we know we're done
|
||||
// with the search and there cannot be any more matches to report.
|
||||
if state.rev_eoi {
|
||||
return Ok(());
|
||||
} else if state.at == input.start() {
|
||||
// At this point, we should follow the EOI transition. This
|
||||
// will cause us the skip the main loop below and fall through
|
||||
// to the final 'eoi_rev' transition.
|
||||
state.rev_eoi = true;
|
||||
} else {
|
||||
// We haven't hit the end of the search yet, so move on.
|
||||
state.at -= 1;
|
||||
}
|
||||
sid
|
||||
}
|
||||
};
|
||||
cache.search_start(state.at);
|
||||
while !state.rev_eoi {
|
||||
sid = dfa
|
||||
.next_state(cache, sid, input.haystack()[state.at])
|
||||
.map_err(|_| gave_up(state.at))?;
|
||||
if sid.is_tagged() {
|
||||
state.id = Some(sid);
|
||||
if sid.is_start() {
|
||||
// do nothing
|
||||
} else if sid.is_match() {
|
||||
state.next_match_index = Some(1);
|
||||
let pattern = dfa.match_pattern(cache, sid, 0);
|
||||
state.mat = Some(HalfMatch::new(pattern, state.at + 1));
|
||||
cache.search_finish(state.at);
|
||||
return Ok(());
|
||||
} else if sid.is_dead() {
|
||||
cache.search_finish(state.at);
|
||||
return Ok(());
|
||||
} else if sid.is_quit() {
|
||||
cache.search_finish(state.at);
|
||||
return Err(MatchError::quit(
|
||||
input.haystack()[state.at],
|
||||
state.at,
|
||||
));
|
||||
} else {
|
||||
debug_assert!(sid.is_unknown());
|
||||
unreachable!("sid being unknown is a bug");
|
||||
}
|
||||
}
|
||||
if state.at == input.start() {
|
||||
break;
|
||||
}
|
||||
state.at -= 1;
|
||||
cache.search_update(state.at);
|
||||
}
|
||||
|
||||
let result = eoi_rev(dfa, cache, input, &mut sid, &mut state.mat);
|
||||
state.rev_eoi = true;
|
||||
state.id = Some(sid);
|
||||
if state.mat.is_some() {
|
||||
// '1' is always correct here since if we get to this point, this
|
||||
// always corresponds to the first (index '0') match discovered at
|
||||
// this position. So the next match to report at this position (if
|
||||
// it exists) is at index '1'.
|
||||
state.next_match_index = Some(1);
|
||||
}
|
||||
cache.search_finish(input.start());
|
||||
result
|
||||
}
|
||||
|
||||
#[cfg_attr(feature = "perf-inline", inline(always))]
|
||||
fn init_fwd(
|
||||
dfa: &DFA,
|
||||
cache: &mut Cache,
|
||||
input: &Input<'_>,
|
||||
) -> Result<LazyStateID, MatchError> {
|
||||
let sid = dfa.start_state_forward(cache, input)?;
|
||||
// Start states can never be match states, since all matches are delayed
|
||||
// by 1 byte.
|
||||
debug_assert!(!sid.is_match());
|
||||
Ok(sid)
|
||||
}
|
||||
|
||||
#[cfg_attr(feature = "perf-inline", inline(always))]
|
||||
fn init_rev(
|
||||
dfa: &DFA,
|
||||
cache: &mut Cache,
|
||||
input: &Input<'_>,
|
||||
) -> Result<LazyStateID, MatchError> {
|
||||
let sid = dfa.start_state_reverse(cache, input)?;
|
||||
// Start states can never be match states, since all matches are delayed
|
||||
// by 1 byte.
|
||||
debug_assert!(!sid.is_match());
|
||||
Ok(sid)
|
||||
}
|
||||
|
||||
#[cfg_attr(feature = "perf-inline", inline(always))]
|
||||
fn eoi_fwd(
|
||||
dfa: &DFA,
|
||||
cache: &mut Cache,
|
||||
input: &Input<'_>,
|
||||
sid: &mut LazyStateID,
|
||||
mat: &mut Option<HalfMatch>,
|
||||
) -> Result<(), MatchError> {
|
||||
let sp = input.get_span();
|
||||
match input.haystack().get(sp.end) {
|
||||
Some(&b) => {
|
||||
*sid =
|
||||
dfa.next_state(cache, *sid, b).map_err(|_| gave_up(sp.end))?;
|
||||
if sid.is_match() {
|
||||
let pattern = dfa.match_pattern(cache, *sid, 0);
|
||||
*mat = Some(HalfMatch::new(pattern, sp.end));
|
||||
} else if sid.is_quit() {
|
||||
return Err(MatchError::quit(b, sp.end));
|
||||
}
|
||||
}
|
||||
None => {
|
||||
*sid = dfa
|
||||
.next_eoi_state(cache, *sid)
|
||||
.map_err(|_| gave_up(input.haystack().len()))?;
|
||||
if sid.is_match() {
|
||||
let pattern = dfa.match_pattern(cache, *sid, 0);
|
||||
*mat = Some(HalfMatch::new(pattern, input.haystack().len()));
|
||||
}
|
||||
// N.B. We don't have to check 'is_quit' here because the EOI
|
||||
// transition can never lead to a quit state.
|
||||
debug_assert!(!sid.is_quit());
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg_attr(feature = "perf-inline", inline(always))]
|
||||
fn eoi_rev(
|
||||
dfa: &DFA,
|
||||
cache: &mut Cache,
|
||||
input: &Input<'_>,
|
||||
sid: &mut LazyStateID,
|
||||
mat: &mut Option<HalfMatch>,
|
||||
) -> Result<(), MatchError> {
|
||||
let sp = input.get_span();
|
||||
if sp.start > 0 {
|
||||
let byte = input.haystack()[sp.start - 1];
|
||||
*sid = dfa
|
||||
.next_state(cache, *sid, byte)
|
||||
.map_err(|_| gave_up(sp.start))?;
|
||||
if sid.is_match() {
|
||||
let pattern = dfa.match_pattern(cache, *sid, 0);
|
||||
*mat = Some(HalfMatch::new(pattern, sp.start));
|
||||
} else if sid.is_quit() {
|
||||
return Err(MatchError::quit(byte, sp.start - 1));
|
||||
}
|
||||
} else {
|
||||
*sid =
|
||||
dfa.next_eoi_state(cache, *sid).map_err(|_| gave_up(sp.start))?;
|
||||
if sid.is_match() {
|
||||
let pattern = dfa.match_pattern(cache, *sid, 0);
|
||||
*mat = Some(HalfMatch::new(pattern, 0));
|
||||
}
|
||||
// N.B. We don't have to check 'is_quit' here because the EOI
|
||||
// transition can never lead to a quit state.
|
||||
debug_assert!(!sid.is_quit());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Re-compute the starting state that a DFA should be in after finding a
|
||||
/// prefilter candidate match at the position `at`.
|
||||
///
|
||||
/// It is always correct to call this, but not always necessary. Namely,
|
||||
/// whenever the DFA has a universal start state, the DFA can remain in the
|
||||
/// start state that it was in when it ran the prefilter. Why? Because in that
|
||||
/// case, there is only one start state.
|
||||
///
|
||||
/// When does a DFA have a universal start state? In precisely cases where
|
||||
/// it has no look-around assertions in its prefix. So for example, `\bfoo`
|
||||
/// does not have a universal start state because the start state depends on
|
||||
/// whether the byte immediately before the start position is a word byte or
|
||||
/// not. However, `foo\b` does have a universal start state because the word
|
||||
/// boundary does not appear in the pattern's prefix.
|
||||
///
|
||||
/// So... most cases don't need this, but when a pattern doesn't have a
|
||||
/// universal start state, then after a prefilter candidate has been found, the
|
||||
/// current state *must* be re-litigated as if computing the start state at the
|
||||
/// beginning of the search because it might change. That is, not all start
|
||||
/// states are created equal.
|
||||
///
|
||||
/// Why avoid it? Because while it's not super expensive, it isn't a trivial
|
||||
/// operation to compute the start state. It is much better to avoid it and
|
||||
/// just state in the current state if you know it to be correct.
|
||||
#[cfg_attr(feature = "perf-inline", inline(always))]
|
||||
fn prefilter_restart(
|
||||
dfa: &DFA,
|
||||
cache: &mut Cache,
|
||||
input: &Input<'_>,
|
||||
at: usize,
|
||||
) -> Result<LazyStateID, MatchError> {
|
||||
let mut input = input.clone();
|
||||
input.set_start(at);
|
||||
init_fwd(dfa, cache, &input)
|
||||
}
|
||||
|
||||
/// A convenience routine for constructing a "gave up" match error.
|
||||
#[cfg_attr(feature = "perf-inline", inline(always))]
|
||||
fn gave_up(offset: usize) -> MatchError {
|
||||
MatchError::gave_up(offset)
|
||||
}
|
||||
Reference in new issue
Block a user